Bug 774106 - Don't use Makefile.in to populate virtualenv; r=glandium
authorGregory Szorc <gps@mozilla.com>
Mon, 23 Jul 2012 00:19:30 -0700
changeset 101940 fda9279d7031f4a7100c2ac6421e4bf26f72e0d8
parent 101939 7295d036a37b691f3b7e75093784bad7265f4d68
child 101941 c9149702e8106b1cdcd93b24c82e3c86a2dc6d96
push id18
push usershu@rfrn.org
push dateMon, 06 Aug 2012 22:42:45 +0000
reviewersglandium
bugs774106
milestone17.0a1
Bug 774106 - Don't use Makefile.in to populate virtualenv; r=glandium
allmakefiles.sh
build/virtualenv/Makefile.in
build/virtualenv/packages.txt
build/virtualenv/populate_virtualenv.py
client.mk
configure.in
--- a/allmakefiles.sh
+++ b/allmakefiles.sh
@@ -25,17 +25,16 @@ fi
 
 # Common makefiles used by everyone
 add_makefiles "
 Makefile
 build/Makefile
 build/pgo/Makefile
 build/pgo/blueprint/Makefile
 build/pgo/js-input/Makefile
-build/virtualenv/Makefile
 config/Makefile
 config/autoconf.mk
 config/nspr/Makefile
 config/doxygen.cfg
 config/expandlibs_config.py
 mfbt/Makefile
 probes/Makefile
 extensions/Makefile
deleted file mode 100644
--- a/build/virtualenv/Makefile.in
+++ /dev/null
@@ -1,35 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-DEPTH     = ../..
-topsrcdir = @top_srcdir@
-srcdir    = @srcdir@
-VPATH     = @srcdir@
-
-include $(DEPTH)/config/autoconf.mk
-
-# Paths here are topsrcdir-relative, and
-# must be in dependency-order.
-setuptools_packages := \
-  other-licenses/simplejson-2.1.1 \
-  testing/mozbase/manifestdestiny \
-  testing/mozbase/mozinfo \
-  testing/mozbase/mozinstall \
-  testing/mozbase/mozlog \
-  testing/mozbase/mozprocess \
-  testing/mozbase/mozprofile \
-  testing/mozbase/mozrunner \
-  build/pylib/blessings \
-  $(NULL)
-
-
-define install_setuptools_package
-cd $(topsrcdir)/$(1)/; CFLAGS="$(HOST_CFLAGS)" LDFLAGS="$(HOST_LDFLAGS)" CXXFLAGS="$(HOST_CXXFLAGS)" $(PYTHON) setup.py develop
-
-endef
-
-default::
-	$(foreach package,$(setuptools_packages),$(call install_setuptools_package,$(package)))
-
-include $(topsrcdir)/config/rules.mk
new file mode 100644
--- /dev/null
+++ b/build/virtualenv/packages.txt
@@ -0,0 +1,9 @@
+setup.py:other-licenses/simplejson-2.1.1:develop
+setup.py:testing/mozbase/manifestdestiny:develop
+setup.py:testing/mozbase/mozinfo:develop
+setup.py:testing/mozbase/mozinstall:develop
+setup.py:testing/mozbase/mozlog:develop
+setup.py:testing/mozbase/mozprocess:develop
+setup.py:testing/mozbase/mozprofile:develop
+setup.py:testing/mozbase/mozrunner:develop
+setup.py:build/pylib/blessings:develop
new file mode 100755
--- /dev/null
+++ b/build/virtualenv/populate_virtualenv.py
@@ -0,0 +1,63 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains code for populating the virtualenv environment for
+# Mozilla's build system. It is typically called as part of configure.
+
+import os.path
+import subprocess
+import sys
+
+def populate_virtualenv(top_source_directory, manifest_filename):
+    """Populate the virtualenv from the contents of a manifest.
+
+    The manifest file consists of colon-delimited fields. The first field
+    specifies the action. The remaining fields are arguments to that action.
+    The following actions are supported:
+
+      setup.py -- Invoke setup.py for a package. Expects the arguments:
+        1. relative path directory containing setup.py.
+        2. argument(s) to setup.py. e.g. "develop". Each program argument is
+           delimited by a colon. Arguments with colons are not yet supported.
+
+    Note that the Python interpreter running this function should be the one
+    from the virtualenv. If it is the system Python or if the environment is
+    not configured properly, packages could be installed into the wrong place.
+    This is how virtualenv's work.
+    """
+    packages = []
+    fh = open(manifest_filename, 'rU')
+    for line in fh:
+        packages.append(line.rstrip().split(':'))
+    fh.close()
+
+    for package in packages:
+        if package[0] == 'setup.py':
+            assert len(package) >= 2
+
+            call_setup(os.path.join(top_source_directory, package[1]),
+                package[2:])
+
+def call_setup(directory, arguments):
+    """Calls setup.py in a directory."""
+    setup = os.path.join(directory, 'setup.py')
+
+    program = [sys.executable, setup]
+    program.extend(arguments)
+
+    # We probably could call the contents of this file inside the context of
+    # this interpreter using execfile() or similar. However, if global
+    # variables like sys.path are adjusted, this could cause all kinds of
+    # havoc. While this may work, invoking a new process is safer.
+    result = subprocess.call(program, cwd=directory)
+
+    if result != 0:
+        raise Exception('Error installing package: %s' % directory)
+
+# configure invokes us with /path/to/topsrcdir and /path/to/manifest
+if __name__ == '__main__':
+    assert len(sys.argv) == 3
+
+    populate_virtualenv(sys.argv[1], sys.argv[2])
+    sys.exit(0)
--- a/client.mk
+++ b/client.mk
@@ -249,37 +249,37 @@ else
 ####################################
 # Configure
 
 MAKEFILE      = $(wildcard $(OBJDIR)/Makefile)
 CONFIG_STATUS = $(wildcard $(OBJDIR)/config.status)
 CONFIG_CACHE  = $(wildcard $(OBJDIR)/config.cache)
 
 EXTRA_CONFIG_DEPS := \
-	$(TOPSRCDIR)/aclocal.m4 \
-	$(wildcard $(TOPSRCDIR)/build/autoconf/*.m4) \
-	$(TOPSRCDIR)/js/src/aclocal.m4 \
-	$(NULL)
+  $(TOPSRCDIR)/aclocal.m4 \
+  $(wildcard $(TOPSRCDIR)/build/autoconf/*.m4) \
+  $(TOPSRCDIR)/js/src/aclocal.m4 \
+  $(NULL)
 
 $(CONFIGURES): %: %.in $(EXTRA_CONFIG_DEPS)
 	@$(PYTHON) $(TOPSRCDIR)/js/src/config/check-sync-dirs.py $(TOPSRCDIR)/js/src/build $(TOPSRCDIR)/build
 	@echo Generating $@ using autoconf
 	cd $(@D); $(AUTOCONF)
 
 CONFIG_STATUS_DEPS := \
-	$(wildcard \
-        $(CONFIGURES) \
-        $(TOPSRCDIR)/allmakefiles.sh \
-        $(TOPSRCDIR)/nsprpub/configure \
-        $(TOPSRCDIR)/config/milestone.txt \
-        $(TOPSRCDIR)/js/src/config/milestone.txt \
-        $(TOPSRCDIR)/browser/config/version.txt \
-        $(TOPSRCDIR)/*/confvars.sh \
-	) \
-	$(NULL)
+  $(wildcard $(TOPSRCDIR)/*/confvars.sh) \
+  $(CONFIGURES) \
+  $(TOPSRCDIR)/allmakefiles.sh \
+  $(TOPSRCDIR)/nsprpub/configure \
+  $(TOPSRCDIR)/config/milestone.txt \
+  $(TOPSRCDIR)/js/src/config/milestone.txt \
+  $(TOPSRCDIR)/browser/config/version.txt \
+  $(TOPSRCDIR)/build/virtualenv/packages.txt \
+  $(TOPSRCDIR)/build/virtualenv/populate_virtualenv.py \
+  $(NULL)
 
 CONFIGURE_ENV_ARGS += \
   MAKE="$(MAKE)" \
   $(NULL)
 
 # configure uses the program name to determine @srcdir@. Calling it without
 #   $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full
 #   path of $(TOPSRCDIR).
--- a/configure.in
+++ b/configure.in
@@ -8775,16 +8775,25 @@ mingw*)
     ;;
 *)
     PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
     ;;
 esac
 
 AC_SUBST(PYTHON)
 
+# Populate the virtualenv
+AC_MSG_RESULT([Populating Python virtualenv])
+MACOSX_DEPLOYMENT_TARGET= LDFLAGS="${HOST_LDFLAGS}" \
+  CC="${CC}" CXX="${CXX}" \
+  CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" \
+  $PYTHON $_topsrcdir/build/virtualenv/populate_virtualenv.py \
+    $_topsrcdir $_topsrcdir/build/virtualenv/packages.txt \
+  || exit 1
+
 dnl Load the list of Makefiles to generate.
 dnl   To add new Makefiles, edit allmakefiles.sh.
 dnl   allmakefiles.sh sets the variable, MAKEFILES.
 . ${srcdir}/allmakefiles.sh
 dnl
 dnl Run a perl script to quickly create the makefiles.
 dnl If it succeeds, it outputs a shell command to set CONFIG_FILES
 dnl   for the files it cannot handle correctly. This way, config.status
@@ -8848,20 +8857,16 @@ if test -n "$MOZ_WEBRTC"; then
      $GYP_WEBRTC_OPTIONS \
      --generator-output=${_objdir}/media/webrtc/trunk/testing/ \
      ${srcdir}/media/webrtc/trunk/testing/gtest.gyp
    if test "$?" != 0; then
       AC_MSG_ERROR([failed to generate gtest Makefiles])
    fi
 fi
 
-# Populate the virtualenv
-AC_MSG_RESULT([Populating Python virtualenv])
-$MAKE -C build/virtualenv MACOSX_DEPLOYMENT_TARGET=  || exit 1
-
 # Generate a JSON config file for unittest harnesses etc to read
 # build configuration details from in a standardized way.
 OS_TARGET=${OS_TARGET} TARGET_CPU=${TARGET_CPU} MOZ_DEBUG=${MOZ_DEBUG} \
 MOZ_WIDGET_TOOLKIT=${MOZ_WIDGET_TOOLKIT} UNIVERSAL_BINARY=${UNIVERSAL_BINARY} \
   $PYTHON ${_topsrcdir}/config/writemozinfo.py ./mozinfo.json.tmp
 if cmp -s ./mozinfo.json.tmp ./mozinfo.json; then
   rm ./mozinfo.json.tmp
 else