Bug 774106 - Don't use Makefile.in to populate virtualenv; r=glandium
authorGregory Szorc <gps@mozilla.com>
Mon, 23 Jul 2012 00:19:30 -0700
changeset 106112 fda9279d7031f4a7100c2ac6421e4bf26f72e0d8
parent 106111 7295d036a37b691f3b7e75093784bad7265f4d68
child 106113 c9149702e8106b1cdcd93b24c82e3c86a2dc6d96
push id214
push userakeybl@mozilla.com
push dateWed, 14 Nov 2012 20:38:59 +0000
treeherdermozilla-release@c8b08ec8e1aa [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs774106
milestone17.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 774106 - Don't use Makefile.in to populate virtualenv; r=glandium
allmakefiles.sh
build/virtualenv/Makefile.in
build/virtualenv/packages.txt
build/virtualenv/populate_virtualenv.py
client.mk
configure.in
--- a/allmakefiles.sh
+++ b/allmakefiles.sh
@@ -25,17 +25,16 @@ fi
 
 # Common makefiles used by everyone
 add_makefiles "
 Makefile
 build/Makefile
 build/pgo/Makefile
 build/pgo/blueprint/Makefile
 build/pgo/js-input/Makefile
-build/virtualenv/Makefile
 config/Makefile
 config/autoconf.mk
 config/nspr/Makefile
 config/doxygen.cfg
 config/expandlibs_config.py
 mfbt/Makefile
 probes/Makefile
 extensions/Makefile
deleted file mode 100644
--- a/build/virtualenv/Makefile.in
+++ /dev/null
@@ -1,35 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-DEPTH     = ../..
-topsrcdir = @top_srcdir@
-srcdir    = @srcdir@
-VPATH     = @srcdir@
-
-include $(DEPTH)/config/autoconf.mk
-
-# Paths here are topsrcdir-relative, and
-# must be in dependency-order.
-setuptools_packages := \
-  other-licenses/simplejson-2.1.1 \
-  testing/mozbase/manifestdestiny \
-  testing/mozbase/mozinfo \
-  testing/mozbase/mozinstall \
-  testing/mozbase/mozlog \
-  testing/mozbase/mozprocess \
-  testing/mozbase/mozprofile \
-  testing/mozbase/mozrunner \
-  build/pylib/blessings \
-  $(NULL)
-
-
-define install_setuptools_package
-cd $(topsrcdir)/$(1)/; CFLAGS="$(HOST_CFLAGS)" LDFLAGS="$(HOST_LDFLAGS)" CXXFLAGS="$(HOST_CXXFLAGS)" $(PYTHON) setup.py develop
-
-endef
-
-default::
-	$(foreach package,$(setuptools_packages),$(call install_setuptools_package,$(package)))
-
-include $(topsrcdir)/config/rules.mk
new file mode 100644
--- /dev/null
+++ b/build/virtualenv/packages.txt
@@ -0,0 +1,9 @@
+setup.py:other-licenses/simplejson-2.1.1:develop
+setup.py:testing/mozbase/manifestdestiny:develop
+setup.py:testing/mozbase/mozinfo:develop
+setup.py:testing/mozbase/mozinstall:develop
+setup.py:testing/mozbase/mozlog:develop
+setup.py:testing/mozbase/mozprocess:develop
+setup.py:testing/mozbase/mozprofile:develop
+setup.py:testing/mozbase/mozrunner:develop
+setup.py:build/pylib/blessings:develop
new file mode 100755
--- /dev/null
+++ b/build/virtualenv/populate_virtualenv.py
@@ -0,0 +1,63 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains code for populating the virtualenv environment for
+# Mozilla's build system. It is typically called as part of configure.
+
+import os.path
+import subprocess
+import sys
+
+def populate_virtualenv(top_source_directory, manifest_filename):
+    """Populate the virtualenv from the contents of a manifest.
+
+    The manifest file consists of colon-delimited fields. The first field
+    specifies the action. The remaining fields are arguments to that action.
+    The following actions are supported:
+
+      setup.py -- Invoke setup.py for a package. Expects the arguments:
+        1. relative path directory containing setup.py.
+        2. argument(s) to setup.py. e.g. "develop". Each program argument is
+           delimited by a colon. Arguments with colons are not yet supported.
+
+    Note that the Python interpreter running this function should be the one
+    from the virtualenv. If it is the system Python or if the environment is
+    not configured properly, packages could be installed into the wrong place.
+    This is how virtualenv's work.
+    """
+    packages = []
+    fh = open(manifest_filename, 'rU')
+    for line in fh:
+        packages.append(line.rstrip().split(':'))
+    fh.close()
+
+    for package in packages:
+        if package[0] == 'setup.py':
+            assert len(package) >= 2
+
+            call_setup(os.path.join(top_source_directory, package[1]),
+                package[2:])
+
+def call_setup(directory, arguments):
+    """Calls setup.py in a directory."""
+    setup = os.path.join(directory, 'setup.py')
+
+    program = [sys.executable, setup]
+    program.extend(arguments)
+
+    # We probably could call the contents of this file inside the context of
+    # this interpreter using execfile() or similar. However, if global
+    # variables like sys.path are adjusted, this could cause all kinds of
+    # havoc. While this may work, invoking a new process is safer.
+    result = subprocess.call(program, cwd=directory)
+
+    if result != 0:
+        raise Exception('Error installing package: %s' % directory)
+
+# configure invokes us with /path/to/topsrcdir and /path/to/manifest
+if __name__ == '__main__':
+    assert len(sys.argv) == 3
+
+    populate_virtualenv(sys.argv[1], sys.argv[2])
+    sys.exit(0)
--- a/client.mk
+++ b/client.mk
@@ -249,37 +249,37 @@ else
 ####################################
 # Configure
 
 MAKEFILE      = $(wildcard $(OBJDIR)/Makefile)
 CONFIG_STATUS = $(wildcard $(OBJDIR)/config.status)
 CONFIG_CACHE  = $(wildcard $(OBJDIR)/config.cache)
 
 EXTRA_CONFIG_DEPS := \
-	$(TOPSRCDIR)/aclocal.m4 \
-	$(wildcard $(TOPSRCDIR)/build/autoconf/*.m4) \
-	$(TOPSRCDIR)/js/src/aclocal.m4 \
-	$(NULL)
+  $(TOPSRCDIR)/aclocal.m4 \
+  $(wildcard $(TOPSRCDIR)/build/autoconf/*.m4) \
+  $(TOPSRCDIR)/js/src/aclocal.m4 \
+  $(NULL)
 
 $(CONFIGURES): %: %.in $(EXTRA_CONFIG_DEPS)
 	@$(PYTHON) $(TOPSRCDIR)/js/src/config/check-sync-dirs.py $(TOPSRCDIR)/js/src/build $(TOPSRCDIR)/build
 	@echo Generating $@ using autoconf
 	cd $(@D); $(AUTOCONF)
 
 CONFIG_STATUS_DEPS := \
-	$(wildcard \
-        $(CONFIGURES) \
-        $(TOPSRCDIR)/allmakefiles.sh \
-        $(TOPSRCDIR)/nsprpub/configure \
-        $(TOPSRCDIR)/config/milestone.txt \
-        $(TOPSRCDIR)/js/src/config/milestone.txt \
-        $(TOPSRCDIR)/browser/config/version.txt \
-        $(TOPSRCDIR)/*/confvars.sh \
-	) \
-	$(NULL)
+  $(wildcard $(TOPSRCDIR)/*/confvars.sh) \
+  $(CONFIGURES) \
+  $(TOPSRCDIR)/allmakefiles.sh \
+  $(TOPSRCDIR)/nsprpub/configure \
+  $(TOPSRCDIR)/config/milestone.txt \
+  $(TOPSRCDIR)/js/src/config/milestone.txt \
+  $(TOPSRCDIR)/browser/config/version.txt \
+  $(TOPSRCDIR)/build/virtualenv/packages.txt \
+  $(TOPSRCDIR)/build/virtualenv/populate_virtualenv.py \
+  $(NULL)
 
 CONFIGURE_ENV_ARGS += \
   MAKE="$(MAKE)" \
   $(NULL)
 
 # configure uses the program name to determine @srcdir@. Calling it without
 #   $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full
 #   path of $(TOPSRCDIR).
--- a/configure.in
+++ b/configure.in
@@ -8775,16 +8775,25 @@ mingw*)
     ;;
 *)
     PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
     ;;
 esac
 
 AC_SUBST(PYTHON)
 
+# Populate the virtualenv
+AC_MSG_RESULT([Populating Python virtualenv])
+MACOSX_DEPLOYMENT_TARGET= LDFLAGS="${HOST_LDFLAGS}" \
+  CC="${CC}" CXX="${CXX}" \
+  CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" \
+  $PYTHON $_topsrcdir/build/virtualenv/populate_virtualenv.py \
+    $_topsrcdir $_topsrcdir/build/virtualenv/packages.txt \
+  || exit 1
+
 dnl Load the list of Makefiles to generate.
 dnl   To add new Makefiles, edit allmakefiles.sh.
 dnl   allmakefiles.sh sets the variable, MAKEFILES.
 . ${srcdir}/allmakefiles.sh
 dnl
 dnl Run a perl script to quickly create the makefiles.
 dnl If it succeeds, it outputs a shell command to set CONFIG_FILES
 dnl   for the files it cannot handle correctly. This way, config.status
@@ -8848,20 +8857,16 @@ if test -n "$MOZ_WEBRTC"; then
      $GYP_WEBRTC_OPTIONS \
      --generator-output=${_objdir}/media/webrtc/trunk/testing/ \
      ${srcdir}/media/webrtc/trunk/testing/gtest.gyp
    if test "$?" != 0; then
       AC_MSG_ERROR([failed to generate gtest Makefiles])
    fi
 fi
 
-# Populate the virtualenv
-AC_MSG_RESULT([Populating Python virtualenv])
-$MAKE -C build/virtualenv MACOSX_DEPLOYMENT_TARGET=  || exit 1
-
 # Generate a JSON config file for unittest harnesses etc to read
 # build configuration details from in a standardized way.
 OS_TARGET=${OS_TARGET} TARGET_CPU=${TARGET_CPU} MOZ_DEBUG=${MOZ_DEBUG} \
 MOZ_WIDGET_TOOLKIT=${MOZ_WIDGET_TOOLKIT} UNIVERSAL_BINARY=${UNIVERSAL_BINARY} \
   $PYTHON ${_topsrcdir}/config/writemozinfo.py ./mozinfo.json.tmp
 if cmp -s ./mozinfo.json.tmp ./mozinfo.json; then
   rm ./mozinfo.json.tmp
 else