Bug 802210 - Refactor virtualenv integration; r=glandium
authorGregory Szorc <gps@mozilla.com>
Mon, 22 Oct 2012 10:41:36 -0700
changeset 111054 1423fc67d73ed2c4e40dfa5938898cd336af76f5
parent 111053 67b6f14cd9fb092f2b8d0adb06ccbd60a11604cc
child 111055 c2c998bad111769302587fb90ee436fa3956a599
push id23726
push userryanvm@gmail.com
push dateTue, 23 Oct 2012 01:41:58 +0000
treeherdermozilla-central@48502b61a63e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs802210
milestone19.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 802210 - Refactor virtualenv integration; r=glandium We now populate the virtualenv at the beginning of configure. We have also refactored how the virtualenv is populated. populate_virtualenv.py is completely refactored. Its default action now takes the topsrcdir and virtualenv paths and ensures a virtualenv is created, populated, and up-to-date. If it is out of date, it repopulates it. populate_virtualenv.py also now performs the Python version check validation instead of configure. It's easier to manage in Python than to have configure do it.
build/virtualenv/populate_virtualenv.py
configure.in
--- a/build/virtualenv/populate_virtualenv.py
+++ b/build/virtualenv/populate_virtualenv.py
@@ -1,110 +1,315 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # This file contains code for populating the virtualenv environment for
 # Mozilla's build system. It is typically called as part of configure.
 
-from __future__ import with_statement
+from __future__ import print_function, unicode_literals, with_statement
+
+import distutils.sysconfig
 import os
 import shutil
 import subprocess
 import sys
-import distutils.sysconfig
 
-def populate_virtualenv(top_source_directory, manifest_filename, log_handle):
-    """Populate the virtualenv from the contents of a manifest.
-
-    The manifest file consists of colon-delimited fields. The first field
-    specifies the action. The remaining fields are arguments to that action.
-    The following actions are supported:
-
-      setup.py -- Invoke setup.py for a package. Expects the arguments:
-        1. relative path directory containing setup.py.
-        2. argument(s) to setup.py. e.g. "develop". Each program argument is
-           delimited by a colon. Arguments with colons are not yet supported.
-
-      filename.pth -- Adds the path given as argument to filename.pth under
-          the virtualenv site packages directory.
-
-      optional -- This denotes the action as optional. The requested action
-          is attempted. If it fails, we issue a warning and go on. The initial
-          "optional" field is stripped then the remaining line is processed
-          like normal. e.g. "optional:setup.py:python/foo:built_ext:-i"
-
-      copy -- Copies the given file in the virtualenv site packages directory.
-
-    Note that the Python interpreter running this function should be the one
-    from the virtualenv. If it is the system Python or if the environment is
-    not configured properly, packages could be installed into the wrong place.
-    This is how virtualenv's work.
-    """
-    packages = []
-    fh = open(manifest_filename, 'rU')
-    for line in fh:
-        packages.append(line.rstrip().split(':'))
-    fh.close()
 
-    def handle_package(package):
-        python_lib = distutils.sysconfig.get_python_lib()
-        if package[0] == 'setup.py':
-            assert len(package) >= 2
-
-            call_setup(os.path.join(top_source_directory, package[1]),
-                package[2:])
-
-            return True
-
-        if package[0] == 'copy':
-            assert len(package) == 2
-
-            shutil.copy(os.path.join(top_source_directory, package[1]),
-                        os.path.join(python_lib, os.path.basename(package[1])))
-
-            return True
-
-        if package[0].endswith('.pth'):
-            assert len(package) == 2
-
-            with open(os.path.join(python_lib, package[0]), 'a') as f:
-                f.write("%s\n" % os.path.join(top_source_directory, package[1]))
-
-            return True
-
-        if package[0] == 'optional':
-            try:
-                handle_package(package[1:])
-                return True
-            except:
-                print >>log_handle, 'Error processing command. Ignoring', \
-                    'because optional. (%s)' % ':'.join(package)
-                return False
-
-        raise Exception('Unknown action: %s' % package[0])
-
-    for package in packages:
-        handle_package(package)
+# Minimum version of Python required to build.
+MINIMUM_PYTHON_MAJOR = 2
+MINIMUM_PYTHON_MINOR = 5
 
 
-def call_setup(directory, arguments):
-    """Calls setup.py in a directory."""
-    setup = os.path.join(directory, 'setup.py')
+class VirtualenvManager(object):
+    """Contains logic for managing virtualenvs for building the tree."""
+
+    def __init__(self, topsrcdir, virtualenv_path, log_handle):
+        """Create a new manager.
+
+        Each manager is associated with a source directory, a path where you
+        want the virtualenv to be created, and a handle to write output to.
+        """
+        self.topsrcdir = topsrcdir
+        self.virtualenv_root = virtualenv_path
+        self.log_handle = log_handle
+
+    @property
+    def virtualenv_script_path(self):
+        """Path to virtualenv's own populator script."""
+        return os.path.join(self.topsrcdir, 'python', 'virtualenv',
+            'virtualenv.py')
+
+    @property
+    def manifest_path(self):
+        return os.path.join(self.topsrcdir, 'build', 'virtualenv',
+            'packages.txt')
+
+    @property
+    def python_path(self):
+        if sys.platform in ('win32', 'cygwin'):
+            return os.path.join(self.virtualenv_root, 'Scripts', 'python.exe')
+
+        return os.path.join(self.virtualenv_root, 'bin', 'python')
+
+    @property
+    def activate_path(self):
+        if sys.platform in ('win32', 'cygwin'):
+            return os.path.join(self.virtualenv_root, 'Scripts',
+                'activate_this.py')
+
+        return os.path.join(self.virtualenv_root, 'bin', 'activate_this.py')
+
+    def ensure(self):
+        """Ensure the virtualenv is present and up to date.
+
+        If the virtualenv is up to date, this does nothing. Otherwise, it
+        creates and populates the virtualenv as necessary.
+
+        This should be the main API used from this class as it is the
+        highest-level.
+        """
+        deps = [self.manifest_path, __file__]
+
+        if not os.path.exists(self.virtualenv_root) or \
+            not os.path.exists(self.activate_path):
+
+            return self.build()
+
+        activate_mtime = os.path.getmtime(self.activate_path)
+        dep_mtime = max(os.path.getmtime(p) for p in deps)
+
+        if dep_mtime > activate_mtime:
+            return self.build()
+
+        return self.virtualenv_root
+
+    def create(self):
+        """Create a new, empty virtualenv.
+
+        Receives the path to virtualenv's virtualenv.py script (which will be
+        called out to), the path to create the virtualenv in, and a handle to
+        write output to.
+        """
+        args = [sys.executable, self.virtualenv_script_path,
+            '--system-site-packages', self.virtualenv_root]
 
-    program = [sys.executable, setup]
-    program.extend(arguments)
+        result = subprocess.call(args, stdout=self.log_handle,
+            stderr=subprocess.STDOUT)
+
+        if result != 0:
+            raise Exception('Error creating virtualenv.')
+
+        return self.virtualenv_root
+
+    def populate(self):
+        """Populate the virtualenv.
+
+        The manifest file consists of colon-delimited fields. The first field
+        specifies the action. The remaining fields are arguments to that
+        action. The following actions are supported:
+
+        setup.py -- Invoke setup.py for a package. Expects the arguments:
+            1. relative path directory containing setup.py.
+            2. argument(s) to setup.py. e.g. "develop". Each program argument
+               is delimited by a colon. Arguments with colons are not yet
+               supported.
+
+        filename.pth -- Adds the path given as argument to filename.pth under
+            the virtualenv site packages directory.
+
+        optional -- This denotes the action as optional. The requested action
+            is attempted. If it fails, we issue a warning and go on. The
+            initial "optional" field is stripped then the remaining line is
+            processed like normal. e.g.
+            "optional:setup.py:python/foo:built_ext:-i"
+
+        copy -- Copies the given file in the virtualenv site packages
+            directory.
+
+        Note that the Python interpreter running this function should be the
+        one from the virtualenv. If it is the system Python or if the
+        environment is not configured properly, packages could be installed
+        into the wrong place. This is how virtualenv's work.
+        """
+        packages = []
+        fh = open(self.manifest_path, 'rUt')
+        for line in fh:
+            packages.append(line.rstrip().split(':'))
+        fh.close()
+
+        def handle_package(package):
+            python_lib = distutils.sysconfig.get_python_lib()
+            if package[0] == 'setup.py':
+                assert len(package) >= 2
+
+                self.call_setup(os.path.join(self.topsrcdir, package[1]),
+                    package[2:])
+
+                return True
+
+            if package[0] == 'copy':
+                assert len(package) == 2
+
+                src = os.path.join(self.topsrcdir, package[1])
+                dst = os.path.join(python_lib, os.path.basename(package[1]))
+
+                shutil.copy(src, dst)
+
+                return True
+
+            if package[0].endswith('.pth'):
+                assert len(package) == 2
+
+                path = os.path.join(self.topsrcdir, package[1])
+
+                with open(os.path.join(python_lib, package[0]), 'a') as f:
+                    f.write("%s\n" % path)
+
+                return True
 
-    # We probably could call the contents of this file inside the context of
-    # this interpreter using execfile() or similar. However, if global
-    # variables like sys.path are adjusted, this could cause all kinds of
-    # havoc. While this may work, invoking a new process is safer.
-    result = subprocess.call(program, cwd=directory)
+            if package[0] == 'optional':
+                try:
+                    handle_package(package[1:])
+                    return True
+                except:
+                    print('Error processing command. Ignoring', \
+                        'because optional. (%s)' % ':'.join(package),
+                        file=self.log_handle)
+                    return False
+
+            raise Exception('Unknown action: %s' % package[0])
+
+        # We always target the OS X deployment target that Python itself was
+        # built with, regardless of what's in the current environment. If we
+        # don't do # this, we may run into a Python bug. See
+        # http://bugs.python.org/issue9516 and bug 659881.
+        #
+        # Note that this assumes that nothing compiled in the virtualenv is
+        # shipped as part of a distribution. If we do ship anything, the
+        # deployment target here may be different from what's targeted by the
+        # shipping binaries and # virtualenv-produced binaries may fail to
+        # work.
+        #
+        # We also ignore environment variables that may have been altered by
+        # configure or a mozconfig activated in the current shell. We trust
+        # Python is smart enough to find a proper compiler and to use the
+        # proper compiler flags. If it isn't your Python is likely broken.
+        IGNORE_ENV_VARIABLES = ('CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS')
+
+        try:
+            old_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None)
+            sysconfig_target = \
+                distutils.sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
+
+            if sysconfig_target is not None:
+                os.environ['MACOSX_DEPLOYMENT_TARGET'] = sysconfig_target
+
+            old_env_variables = {}
+            for k in IGNORE_ENV_VARIABLES:
+                if k not in os.environ:
+                    continue
+
+                old_env_variables[k] = os.environ[k]
+                del os.environ[k]
+
+            for package in packages:
+                handle_package(package)
+        finally:
+            try:
+                del os.environ['MACOSX_DEPLOYMENT_TARGET']
+            except KeyError:
+                pass
+
+            if old_target is not None:
+                os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target
+
+            for k in old_env_variables:
+                os.environ[k] = old_env_variables[k]
+
+
+    def call_setup(self, directory, arguments):
+        """Calls setup.py in a directory."""
+        setup = os.path.join(directory, 'setup.py')
+
+        program = [sys.executable, setup]
+        program.extend(arguments)
+
+        # We probably could call the contents of this file inside the context
+        # of # this interpreter using execfile() or similar. However, if global
+        # variables like sys.path are adjusted, this could cause all kinds of
+        # havoc. While this may work, invoking a new process is safer.
+        result = subprocess.call(program, cwd=directory)
 
-    if result != 0:
-        raise Exception('Error installing package: %s' % directory)
+        if result != 0:
+            raise Exception('Error installing package: %s' % directory)
+
+    def build(self):
+        """Build a virtualenv per tree conventions.
+
+        This returns the path of the created virtualenv.
+        """
+
+        self.create()
+
+        # We need to populate the virtualenv using the Python executable in
+        # the virtualenv for paths to be proper.
+
+        args = [self.python_path, __file__, 'populate', self.topsrcdir,
+            self.virtualenv_root]
+
+        result = subprocess.call(args, stdout=self.log_handle,
+            stderr=subprocess.STDOUT, cwd=self.topsrcdir)
+
+        if result != 0:
+            raise Exception('Error populating virtualenv.')
+
+        os.utime(self.activate_path, None)
 
-# configure invokes us with /path/to/topsrcdir and /path/to/manifest
+        return self.virtualenv_root
+
+    def activate(self):
+        """Activate the virtualenv in this Python context.
+
+        If you run a random Python script and wish to "activate" the
+        virtualenv, you can simply instantiate an instance of this class
+        and call .ensure() and .activate() to make the virtualenv active.
+        """
+
+        execfile(self.activate_path, dict(__file__=self.activate_path))
+
+
+def verify_python_version(log_handle):
+    """Ensure the current version of Python is sufficient."""
+    major, minor = sys.version_info[:2]
+
+    if major != MINIMUM_PYTHON_MAJOR or minor < MINIMUM_PYTHON_MINOR:
+        log_handle.write('Python %d.%d or greater (but not Python 3) is '
+            'required to build. ' %
+            (MINIMUM_PYTHON_MAJOR, MINIMUM_PYTHON_MINOR))
+        log_handle.write('You are running Python %d.%d.\n' % (major, minor))
+        sys.exit(1)
+
+
 if __name__ == '__main__':
-    assert len(sys.argv) == 3
+    if len(sys.argv) < 3:
+        print('Usage: populate_virtualenv.py /path/to/topsrcdir /path/to/virtualenv')
+        sys.exit(1)
+
+    verify_python_version(sys.stdout)
+
+    topsrcdir = sys.argv[1]
+    virtualenv_path = sys.argv[2]
+    populate = False
 
-    populate_virtualenv(sys.argv[1], sys.argv[2], sys.stdout)
-    sys.exit(0)
+    # This should only be called internally.
+    if sys.argv[1] == 'populate':
+        populate = True
+        topsrcdir = sys.argv[2]
+        virtualenv_path = sys.argv[3]
+
+    manager = VirtualenvManager(topsrcdir, virtualenv_path, sys.stdout)
+
+    if populate:
+        manager.populate()
+    else:
+        manager.ensure()
+
--- a/configure.in
+++ b/configure.in
@@ -57,18 +57,16 @@ MOZJPEG=62
 MOZPNG=10513
 NSPR_VERSION=4
 NSS_VERSION=3
 
 dnl Set the minimum version of toolkit libs used by mozilla
 dnl ========================================================
 GLIB_VERSION=1.2.0
 PERL_VERSION=5.006
-PYTHON_VERSION_MAJOR=2
-PYTHON_VERSION_MINOR=5
 CAIRO_VERSION=1.10
 PANGO_VERSION=1.14.0
 GTK2_VERSION=2.10.0
 WINDRES_VERSION=2.14.90
 W32API_VERSION=3.14
 GNOMEVFS_VERSION=2.0
 GNOMEUI_VERSION=2.2.0
 GCONF_VERSION=1.2.1
@@ -120,16 +118,40 @@ then
 	***
 	EOF
     exit 1
     break
   fi
 fi
 MOZ_BUILD_ROOT=`pwd`
 
+MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python2.6 python2.5 python)
+if test -z "$PYTHON"; then
+    AC_MSG_ERROR([python was not found in \$PATH])
+fi
+
+AC_MSG_RESULT([Creating Python environment])
+dnl This verifies our Python version is sane and ensures the Python
+dnl virtualenv is present and up to date. It sanitizes the environment
+dnl for us, so we don't need to clean anything out.
+$PYTHON $_topsrcdir/build/virtualenv/populate_virtualenv.py \
+    $_topsrcdir $MOZ_BUILD_ROOT/_virtualenv || exit 1
+
+dnl Create a virtualenv where we can install local Python packages
+case "$host_os" in
+mingw*)
+    PYTHON=`cd $MOZ_BUILD_ROOT && pwd -W`/_virtualenv/Scripts/python.exe
+    ;;
+*)
+    PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
+    ;;
+esac
+
+AC_SUBST(PYTHON)
+
 MOZ_DEFAULT_COMPILER
 
 COMPILE_ENVIRONMENT=1
 MOZ_ARG_DISABLE_BOOL(compile-environment,
 [  --disable-compile-environment
                           Disable compiler/library checks.],
     COMPILE_ENVIRONMENT= )
 AC_SUBST(COMPILE_ENVIRONMENT)
@@ -798,21 +820,16 @@ AC_MSG_CHECKING([for full perl installat
 _perl_res=$?
 if test "$_perl_res" != 0; then
     AC_MSG_RESULT([no])
     AC_MSG_ERROR([Cannot find Config.pm or \$Config{archlib}.  A full perl installation is required.])
 else
     AC_MSG_RESULT([yes])
 fi
 
-MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python2.6 python2.5 python)
-if test -z "$PYTHON"; then
-    AC_MSG_ERROR([python was not found in \$PATH])
-fi
-
 MOZ_ARG_WITH_BOOL(system-ply,
 [  --with-system-ply       Use system installed python ply library],
     [if $PYTHON -c 'import ply' 2>&5; then
          MOZ_SYSTEM_PLY=1
      else
          AC_MSG_ERROR([python ply library is not found but --with-system-ply was requested])
      fi])
 
@@ -1834,34 +1851,16 @@ case "$host" in
     ;;
 
 *)
     HOST_CFLAGS="$HOST_CFLAGS -DXP_UNIX"
     HOST_OPTIMIZE_FLAGS="${HOST_OPTIMIZE_FLAGS=-O2}"
     ;;
 esac
 
-dnl We require version 2.5 or newer of Python to build.
-AC_MSG_CHECKING([for Python version >= $PYTHON_VERSION_MAJOR.$PYTHON_VERSION_MINOR but not 3.x])
-
-changequote(:)
-read python_version_major python_version_minor python_version_micro <<EOF
-`$PYTHON -c 'import sys; print sys.version_info[0], sys.version_info[1], sys.version_info[2]'`
-EOF
-changequote([,])
-
-if test $python_version_major -ne $PYTHON_VERSION_MAJOR; then
-    AC_MSG_ERROR([Cannot build on Python $python_version_major.])
-else
-    if test $python_version_minor -lt $PYTHON_VERSION_MINOR; then
-        AC_MSG_ERROR([Cannot build on Python $python_version_major.$python_version_minor. You need at least $PYTHON_VERSION_MAJOR.$PYTHON_VERSION_MINOR.])
-    fi
-fi
-AC_MSG_RESULT([yes])
-
 dnl Check for using a custom <stdint.h> implementation
 dnl ========================================================
 AC_MSG_CHECKING(for custom <stdint.h> implementation)
 if test "$MOZ_CUSTOM_STDINT_H"; then
   AC_DEFINE_UNQUOTED(MOZ_CUSTOM_STDINT_H, "$MOZ_CUSTOM_STDINT_H")
   AC_MSG_RESULT(using $MOZ_CUSTOM_STDINT_H)
 else
   AC_MSG_RESULT(none specified)
@@ -8818,61 +8817,16 @@ case "$host" in
 *-apple-darwin11*)
     FIXED_EGREP="env ARCHPREFERENCE=i386,x86_64 arch egrep"
     ;;
 *)
     FIXED_EGREP="egrep"
     ;;
 esac
 
-dnl Create a virtualenv where we can install local Python packages
-AC_MSG_RESULT([Creating Python virtualenv])
-rm -rf _virtualenv
-mkdir -p _virtualenv
-MACOSX_DEPLOYMENT_TARGET= PYTHONDONTWRITEBYTECODE= $PYTHON $_topsrcdir/python/virtualenv/virtualenv.py --system-site-packages ./_virtualenv
-case "$host_os" in
-mingw*)
-    PYTHON=$MOZ_BUILD_ROOT/_virtualenv/Scripts/python.exe
-    ;;
-*)
-    PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
-    ;;
-esac
-
-AC_SUBST(PYTHON)
-
-dnl Populate the virtualenv
-
-dnl We always use the version Python was compiled with to ensure that compiled
-dnl extensions work properly. This works around a bug in Python. See also
-dnl http://bugs.python.org/issue9516 and bug 659881.
-dnl
-dnl Please note that this assumes nothing built during virtualenv population is
-dnl shipped as part of a release package. If it does ship, binaries built here
-dnl may not be compatible with the minimum supported OS X version.
-osx_deployment_target_system=
-
-if test $python_version_major -ne 2; then
-    AC_MSG_ERROR([Assertion failed: building with Python 2.])
-fi
-
-dnl sysconfig is only present on Python 2.7 and above.
-if test $python_version_minor -ge 7; then
-    osx_deployment_target_system=`$PYTHON -c 'import sysconfig; print sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET")'`
-fi
-
-AC_MSG_RESULT([Populating Python virtualenv])
-MACOSX_DEPLOYMENT_TARGET=$osx_deployment_target_system \
-  LDFLAGS="${HOST_LDFLAGS}" \
-  CC="${HOST_CC}" CXX="${HOST_CXX}" \
-  CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" \
-  $PYTHON $_topsrcdir/build/virtualenv/populate_virtualenv.py \
-    $_topsrcdir $_topsrcdir/build/virtualenv/packages.txt \
-  || exit 1
-
 dnl Load the list of Makefiles to generate.
 dnl   To add new Makefiles, edit allmakefiles.sh.
 dnl   allmakefiles.sh sets the variable, MAKEFILES.
 . ${srcdir}/allmakefiles.sh
 
 echo $MAKEFILES > unallmakefiles
 
 AC_OUTPUT($MAKEFILES)