Bug 802210 - Refactor virtualenv integration; r=glandium
authorGregory Szorc <gps@mozilla.com>
Mon, 22 Oct 2012 10:41:36 -0700
changeset 111186 1423fc67d73ed2c4e40dfa5938898cd336af76f5
parent 111185 67b6f14cd9fb092f2b8d0adb06ccbd60a11604cc
child 111187 c2c998bad111769302587fb90ee436fa3956a599
push id93
push usernmatsakis@mozilla.com
push dateWed, 31 Oct 2012 21:26:57 +0000
reviewersglandium
bugs802210
milestone19.0a1
Bug 802210 - Refactor virtualenv integration; r=glandium We now populate the virtualenv at the beginning of configure. We have also refactored how the virtualenv is populated. populate_virtualenv.py is completely refactored. Its default action now takes the topsrcdir and virtualenv paths and ensures a virtualenv is created, populated, and up-to-date. If it is out of date, it repopulates it. populate_virtualenv.py also now performs the Python version check validation instead of configure. It's easier to manage in Python than to have configure do it.
build/virtualenv/populate_virtualenv.py
configure.in
--- a/build/virtualenv/populate_virtualenv.py
+++ b/build/virtualenv/populate_virtualenv.py
@@ -1,110 +1,315 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # This file contains code for populating the virtualenv environment for
 # Mozilla's build system. It is typically called as part of configure.
 
-from __future__ import with_statement
+from __future__ import print_function, unicode_literals, with_statement
+
+import distutils.sysconfig
 import os
 import shutil
 import subprocess
 import sys
-import distutils.sysconfig
 
-def populate_virtualenv(top_source_directory, manifest_filename, log_handle):
-    """Populate the virtualenv from the contents of a manifest.
-
-    The manifest file consists of colon-delimited fields. The first field
-    specifies the action. The remaining fields are arguments to that action.
-    The following actions are supported:
-
-      setup.py -- Invoke setup.py for a package. Expects the arguments:
-        1. relative path directory containing setup.py.
-        2. argument(s) to setup.py. e.g. "develop". Each program argument is
-           delimited by a colon. Arguments with colons are not yet supported.
-
-      filename.pth -- Adds the path given as argument to filename.pth under
-          the virtualenv site packages directory.
-
-      optional -- This denotes the action as optional. The requested action
-          is attempted. If it fails, we issue a warning and go on. The initial
-          "optional" field is stripped then the remaining line is processed
-          like normal. e.g. "optional:setup.py:python/foo:built_ext:-i"
-
-      copy -- Copies the given file in the virtualenv site packages directory.
-
-    Note that the Python interpreter running this function should be the one
-    from the virtualenv. If it is the system Python or if the environment is
-    not configured properly, packages could be installed into the wrong place.
-    This is how virtualenv's work.
-    """
-    packages = []
-    fh = open(manifest_filename, 'rU')
-    for line in fh:
-        packages.append(line.rstrip().split(':'))
-    fh.close()
 
-    def handle_package(package):
-        python_lib = distutils.sysconfig.get_python_lib()
-        if package[0] == 'setup.py':
-            assert len(package) >= 2
-
-            call_setup(os.path.join(top_source_directory, package[1]),
-                package[2:])
-
-            return True
-
-        if package[0] == 'copy':
-            assert len(package) == 2
-
-            shutil.copy(os.path.join(top_source_directory, package[1]),
-                        os.path.join(python_lib, os.path.basename(package[1])))
-
-            return True
-
-        if package[0].endswith('.pth'):
-            assert len(package) == 2
-
-            with open(os.path.join(python_lib, package[0]), 'a') as f:
-                f.write("%s\n" % os.path.join(top_source_directory, package[1]))
-
-            return True
-
-        if package[0] == 'optional':
-            try:
-                handle_package(package[1:])
-                return True
-            except:
-                print >>log_handle, 'Error processing command. Ignoring', \
-                    'because optional. (%s)' % ':'.join(package)
-                return False
-
-        raise Exception('Unknown action: %s' % package[0])
-
-    for package in packages:
-        handle_package(package)
+# Minimum version of Python required to build.
+MINIMUM_PYTHON_MAJOR = 2
+MINIMUM_PYTHON_MINOR = 5
 
 
-def call_setup(directory, arguments):
-    """Calls setup.py in a directory."""
-    setup = os.path.join(directory, 'setup.py')
+class VirtualenvManager(object):
+    """Contains logic for managing virtualenvs for building the tree."""
+
+    def __init__(self, topsrcdir, virtualenv_path, log_handle):
+        """Create a new manager.
+
+        Each manager is associated with a source directory, a path where you
+        want the virtualenv to be created, and a handle to write output to.
+        """
+        self.topsrcdir = topsrcdir
+        self.virtualenv_root = virtualenv_path
+        self.log_handle = log_handle
+
+    @property
+    def virtualenv_script_path(self):
+        """Path to virtualenv's own populator script."""
+        return os.path.join(self.topsrcdir, 'python', 'virtualenv',
+            'virtualenv.py')
+
+    @property
+    def manifest_path(self):
+        return os.path.join(self.topsrcdir, 'build', 'virtualenv',
+            'packages.txt')
+
+    @property
+    def python_path(self):
+        if sys.platform in ('win32', 'cygwin'):
+            return os.path.join(self.virtualenv_root, 'Scripts', 'python.exe')
+
+        return os.path.join(self.virtualenv_root, 'bin', 'python')
+
+    @property
+    def activate_path(self):
+        if sys.platform in ('win32', 'cygwin'):
+            return os.path.join(self.virtualenv_root, 'Scripts',
+                'activate_this.py')
+
+        return os.path.join(self.virtualenv_root, 'bin', 'activate_this.py')
+
+    def ensure(self):
+        """Ensure the virtualenv is present and up to date.
+
+        If the virtualenv is up to date, this does nothing. Otherwise, it
+        creates and populates the virtualenv as necessary.
+
+        This should be the main API used from this class as it is the
+        highest-level.
+        """
+        deps = [self.manifest_path, __file__]
+
+        if not os.path.exists(self.virtualenv_root) or \
+            not os.path.exists(self.activate_path):
+
+            return self.build()
+
+        activate_mtime = os.path.getmtime(self.activate_path)
+        dep_mtime = max(os.path.getmtime(p) for p in deps)
+
+        if dep_mtime > activate_mtime:
+            return self.build()
+
+        return self.virtualenv_root
+
+    def create(self):
+        """Create a new, empty virtualenv.
+
+        Receives the path to virtualenv's virtualenv.py script (which will be
+        called out to), the path to create the virtualenv in, and a handle to
+        write output to.
+        """
+        args = [sys.executable, self.virtualenv_script_path,
+            '--system-site-packages', self.virtualenv_root]
 
-    program = [sys.executable, setup]
-    program.extend(arguments)
+        result = subprocess.call(args, stdout=self.log_handle,
+            stderr=subprocess.STDOUT)
+
+        if result != 0:
+            raise Exception('Error creating virtualenv.')
+
+        return self.virtualenv_root
+
+    def populate(self):
+        """Populate the virtualenv.
+
+        The manifest file consists of colon-delimited fields. The first field
+        specifies the action. The remaining fields are arguments to that
+        action. The following actions are supported:
+
+        setup.py -- Invoke setup.py for a package. Expects the arguments:
+            1. relative path directory containing setup.py.
+            2. argument(s) to setup.py. e.g. "develop". Each program argument
+               is delimited by a colon. Arguments with colons are not yet
+               supported.
+
+        filename.pth -- Adds the path given as argument to filename.pth under
+            the virtualenv site packages directory.
+
+        optional -- This denotes the action as optional. The requested action
+            is attempted. If it fails, we issue a warning and go on. The
+            initial "optional" field is stripped then the remaining line is
+            processed like normal. e.g.
+            "optional:setup.py:python/foo:built_ext:-i"
+
+        copy -- Copies the given file in the virtualenv site packages
+            directory.
+
+        Note that the Python interpreter running this function should be the
+        one from the virtualenv. If it is the system Python or if the
+        environment is not configured properly, packages could be installed
+        into the wrong place. This is how virtualenv's work.
+        """
+        packages = []
+        fh = open(self.manifest_path, 'rUt')
+        for line in fh:
+            packages.append(line.rstrip().split(':'))
+        fh.close()
+
+        def handle_package(package):
+            python_lib = distutils.sysconfig.get_python_lib()
+            if package[0] == 'setup.py':
+                assert len(package) >= 2
+
+                self.call_setup(os.path.join(self.topsrcdir, package[1]),
+                    package[2:])
+
+                return True
+
+            if package[0] == 'copy':
+                assert len(package) == 2
+
+                src = os.path.join(self.topsrcdir, package[1])
+                dst = os.path.join(python_lib, os.path.basename(package[1]))
+
+                shutil.copy(src, dst)
+
+                return True
+
+            if package[0].endswith('.pth'):
+                assert len(package) == 2
+
+                path = os.path.join(self.topsrcdir, package[1])
+
+                with open(os.path.join(python_lib, package[0]), 'a') as f:
+                    f.write("%s\n" % path)
+
+                return True
 
-    # We probably could call the contents of this file inside the context of
-    # this interpreter using execfile() or similar. However, if global
-    # variables like sys.path are adjusted, this could cause all kinds of
-    # havoc. While this may work, invoking a new process is safer.
-    result = subprocess.call(program, cwd=directory)
+            if package[0] == 'optional':
+                try:
+                    handle_package(package[1:])
+                    return True
+                except:
+                    print('Error processing command. Ignoring', \
+                        'because optional. (%s)' % ':'.join(package),
+                        file=self.log_handle)
+                    return False
+
+            raise Exception('Unknown action: %s' % package[0])
+
+        # We always target the OS X deployment target that Python itself was
+        # built with, regardless of what's in the current environment. If we
+        # don't do # this, we may run into a Python bug. See
+        # http://bugs.python.org/issue9516 and bug 659881.
+        #
+        # Note that this assumes that nothing compiled in the virtualenv is
+        # shipped as part of a distribution. If we do ship anything, the
+        # deployment target here may be different from what's targeted by the
+        # shipping binaries and # virtualenv-produced binaries may fail to
+        # work.
+        #
+        # We also ignore environment variables that may have been altered by
+        # configure or a mozconfig activated in the current shell. We trust
+        # Python is smart enough to find a proper compiler and to use the
+        # proper compiler flags. If it isn't your Python is likely broken.
+        IGNORE_ENV_VARIABLES = ('CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS')
+
+        try:
+            old_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None)
+            sysconfig_target = \
+                distutils.sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
+
+            if sysconfig_target is not None:
+                os.environ['MACOSX_DEPLOYMENT_TARGET'] = sysconfig_target
+
+            old_env_variables = {}
+            for k in IGNORE_ENV_VARIABLES:
+                if k not in os.environ:
+                    continue
+
+                old_env_variables[k] = os.environ[k]
+                del os.environ[k]
+
+            for package in packages:
+                handle_package(package)
+        finally:
+            try:
+                del os.environ['MACOSX_DEPLOYMENT_TARGET']
+            except KeyError:
+                pass
+
+            if old_target is not None:
+                os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target
+
+            for k in old_env_variables:
+                os.environ[k] = old_env_variables[k]
+
+
+    def call_setup(self, directory, arguments):
+        """Calls setup.py in a directory."""
+        setup = os.path.join(directory, 'setup.py')
+
+        program = [sys.executable, setup]
+        program.extend(arguments)
+
+        # We probably could call the contents of this file inside the context
+        # of # this interpreter using execfile() or similar. However, if global
+        # variables like sys.path are adjusted, this could cause all kinds of
+        # havoc. While this may work, invoking a new process is safer.
+        result = subprocess.call(program, cwd=directory)
 
-    if result != 0:
-        raise Exception('Error installing package: %s' % directory)
+        if result != 0:
+            raise Exception('Error installing package: %s' % directory)
+
+    def build(self):
+        """Build a virtualenv per tree conventions.
+
+        This returns the path of the created virtualenv.
+        """
+
+        self.create()
+
+        # We need to populate the virtualenv using the Python executable in
+        # the virtualenv for paths to be proper.
+
+        args = [self.python_path, __file__, 'populate', self.topsrcdir,
+            self.virtualenv_root]
+
+        result = subprocess.call(args, stdout=self.log_handle,
+            stderr=subprocess.STDOUT, cwd=self.topsrcdir)
+
+        if result != 0:
+            raise Exception('Error populating virtualenv.')
+
+        os.utime(self.activate_path, None)
 
-# configure invokes us with /path/to/topsrcdir and /path/to/manifest
+        return self.virtualenv_root
+
+    def activate(self):
+        """Activate the virtualenv in this Python context.
+
+        If you run a random Python script and wish to "activate" the
+        virtualenv, you can simply instantiate an instance of this class
+        and call .ensure() and .activate() to make the virtualenv active.
+        """
+
+        execfile(self.activate_path, dict(__file__=self.activate_path))
+
+
+def verify_python_version(log_handle):
+    """Ensure the current version of Python is sufficient."""
+    major, minor = sys.version_info[:2]
+
+    if major != MINIMUM_PYTHON_MAJOR or minor < MINIMUM_PYTHON_MINOR:
+        log_handle.write('Python %d.%d or greater (but not Python 3) is '
+            'required to build. ' %
+            (MINIMUM_PYTHON_MAJOR, MINIMUM_PYTHON_MINOR))
+        log_handle.write('You are running Python %d.%d.\n' % (major, minor))
+        sys.exit(1)
+
+
 if __name__ == '__main__':
-    assert len(sys.argv) == 3
+    if len(sys.argv) < 3:
+        print('Usage: populate_virtualenv.py /path/to/topsrcdir /path/to/virtualenv')
+        sys.exit(1)
+
+    verify_python_version(sys.stdout)
+
+    topsrcdir = sys.argv[1]
+    virtualenv_path = sys.argv[2]
+    populate = False
 
-    populate_virtualenv(sys.argv[1], sys.argv[2], sys.stdout)
-    sys.exit(0)
+    # This should only be called internally.
+    if sys.argv[1] == 'populate':
+        populate = True
+        topsrcdir = sys.argv[2]
+        virtualenv_path = sys.argv[3]
+
+    manager = VirtualenvManager(topsrcdir, virtualenv_path, sys.stdout)
+
+    if populate:
+        manager.populate()
+    else:
+        manager.ensure()
+
--- a/configure.in
+++ b/configure.in
@@ -57,18 +57,16 @@ MOZJPEG=62
 MOZPNG=10513
 NSPR_VERSION=4
 NSS_VERSION=3
 
 dnl Set the minimum version of toolkit libs used by mozilla
 dnl ========================================================
 GLIB_VERSION=1.2.0
 PERL_VERSION=5.006
-PYTHON_VERSION_MAJOR=2
-PYTHON_VERSION_MINOR=5
 CAIRO_VERSION=1.10
 PANGO_VERSION=1.14.0
 GTK2_VERSION=2.10.0
 WINDRES_VERSION=2.14.90
 W32API_VERSION=3.14
 GNOMEVFS_VERSION=2.0
 GNOMEUI_VERSION=2.2.0
 GCONF_VERSION=1.2.1
@@ -120,16 +118,40 @@ then
 	***
 	EOF
     exit 1
     break
   fi
 fi
 MOZ_BUILD_ROOT=`pwd`
 
+MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python2.6 python2.5 python)
+if test -z "$PYTHON"; then
+    AC_MSG_ERROR([python was not found in \$PATH])
+fi
+
+AC_MSG_RESULT([Creating Python environment])
+dnl This verifies our Python version is sane and ensures the Python
+dnl virtualenv is present and up to date. It sanitizes the environment
+dnl for us, so we don't need to clean anything out.
+$PYTHON $_topsrcdir/build/virtualenv/populate_virtualenv.py \
+    $_topsrcdir $MOZ_BUILD_ROOT/_virtualenv || exit 1
+
+dnl Create a virtualenv where we can install local Python packages
+case "$host_os" in
+mingw*)
+    PYTHON=`cd $MOZ_BUILD_ROOT && pwd -W`/_virtualenv/Scripts/python.exe
+    ;;
+*)
+    PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
+    ;;
+esac
+
+AC_SUBST(PYTHON)
+
 MOZ_DEFAULT_COMPILER
 
 COMPILE_ENVIRONMENT=1
 MOZ_ARG_DISABLE_BOOL(compile-environment,
 [  --disable-compile-environment
                           Disable compiler/library checks.],
     COMPILE_ENVIRONMENT= )
 AC_SUBST(COMPILE_ENVIRONMENT)
@@ -798,21 +820,16 @@ AC_MSG_CHECKING([for full perl installat
 _perl_res=$?
 if test "$_perl_res" != 0; then
     AC_MSG_RESULT([no])
     AC_MSG_ERROR([Cannot find Config.pm or \$Config{archlib}.  A full perl installation is required.])
 else
     AC_MSG_RESULT([yes])
 fi
 
-MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python2.6 python2.5 python)
-if test -z "$PYTHON"; then
-    AC_MSG_ERROR([python was not found in \$PATH])
-fi
-
 MOZ_ARG_WITH_BOOL(system-ply,
 [  --with-system-ply       Use system installed python ply library],
     [if $PYTHON -c 'import ply' 2>&5; then
          MOZ_SYSTEM_PLY=1
      else
          AC_MSG_ERROR([python ply library is not found but --with-system-ply was requested])
      fi])
 
@@ -1834,34 +1851,16 @@ case "$host" in
     ;;
 
 *)
     HOST_CFLAGS="$HOST_CFLAGS -DXP_UNIX"
     HOST_OPTIMIZE_FLAGS="${HOST_OPTIMIZE_FLAGS=-O2}"
     ;;
 esac
 
-dnl We require version 2.5 or newer of Python to build.
-AC_MSG_CHECKING([for Python version >= $PYTHON_VERSION_MAJOR.$PYTHON_VERSION_MINOR but not 3.x])
-
-changequote(:)
-read python_version_major python_version_minor python_version_micro <<EOF
-`$PYTHON -c 'import sys; print sys.version_info[0], sys.version_info[1], sys.version_info[2]'`
-EOF
-changequote([,])
-
-if test $python_version_major -ne $PYTHON_VERSION_MAJOR; then
-    AC_MSG_ERROR([Cannot build on Python $python_version_major.])
-else
-    if test $python_version_minor -lt $PYTHON_VERSION_MINOR; then
-        AC_MSG_ERROR([Cannot build on Python $python_version_major.$python_version_minor. You need at least $PYTHON_VERSION_MAJOR.$PYTHON_VERSION_MINOR.])
-    fi
-fi
-AC_MSG_RESULT([yes])
-
 dnl Check for using a custom <stdint.h> implementation
 dnl ========================================================
 AC_MSG_CHECKING(for custom <stdint.h> implementation)
 if test "$MOZ_CUSTOM_STDINT_H"; then
   AC_DEFINE_UNQUOTED(MOZ_CUSTOM_STDINT_H, "$MOZ_CUSTOM_STDINT_H")
   AC_MSG_RESULT(using $MOZ_CUSTOM_STDINT_H)
 else
   AC_MSG_RESULT(none specified)
@@ -8818,61 +8817,16 @@ case "$host" in
 *-apple-darwin11*)
     FIXED_EGREP="env ARCHPREFERENCE=i386,x86_64 arch egrep"
     ;;
 *)
     FIXED_EGREP="egrep"
     ;;
 esac
 
-dnl Create a virtualenv where we can install local Python packages
-AC_MSG_RESULT([Creating Python virtualenv])
-rm -rf _virtualenv
-mkdir -p _virtualenv
-MACOSX_DEPLOYMENT_TARGET= PYTHONDONTWRITEBYTECODE= $PYTHON $_topsrcdir/python/virtualenv/virtualenv.py --system-site-packages ./_virtualenv
-case "$host_os" in
-mingw*)
-    PYTHON=$MOZ_BUILD_ROOT/_virtualenv/Scripts/python.exe
-    ;;
-*)
-    PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
-    ;;
-esac
-
-AC_SUBST(PYTHON)
-
-dnl Populate the virtualenv
-
-dnl We always use the version Python was compiled with to ensure that compiled
-dnl extensions work properly. This works around a bug in Python. See also
-dnl http://bugs.python.org/issue9516 and bug 659881.
-dnl
-dnl Please note that this assumes nothing built during virtualenv population is
-dnl shipped as part of a release package. If it does ship, binaries built here
-dnl may not be compatible with the minimum supported OS X version.
-osx_deployment_target_system=
-
-if test $python_version_major -ne 2; then
-    AC_MSG_ERROR([Assertion failed: building with Python 2.])
-fi
-
-dnl sysconfig is only present on Python 2.7 and above.
-if test $python_version_minor -ge 7; then
-    osx_deployment_target_system=`$PYTHON -c 'import sysconfig; print sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET")'`
-fi
-
-AC_MSG_RESULT([Populating Python virtualenv])
-MACOSX_DEPLOYMENT_TARGET=$osx_deployment_target_system \
-  LDFLAGS="${HOST_LDFLAGS}" \
-  CC="${HOST_CC}" CXX="${HOST_CXX}" \
-  CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" \
-  $PYTHON $_topsrcdir/build/virtualenv/populate_virtualenv.py \
-    $_topsrcdir $_topsrcdir/build/virtualenv/packages.txt \
-  || exit 1
-
 dnl Load the list of Makefiles to generate.
 dnl   To add new Makefiles, edit allmakefiles.sh.
 dnl   allmakefiles.sh sets the variable, MAKEFILES.
 . ${srcdir}/allmakefiles.sh
 
 echo $MAKEFILES > unallmakefiles
 
 AC_OUTPUT($MAKEFILES)