Bug 1253502 - Move python virtualenv initialization to moz.configure. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 04 Mar 2016 14:31:08 +0900
changeset 287211 33859b1a30fa988aed77d3260daee09ade2c938c
parent 287210 2439b4a424353eb0b3f97ca56717ac317790f24f
child 287212 addd5fc00284182a0ce28d302535266d7ef6faea
push id18077
push userkwierso@gmail.com
push dateWed, 09 Mar 2016 00:09:21 +0000
treeherderfx-team@a67b36e32207 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1253502
milestone47.0a1
Bug 1253502 - Move python virtualenv initialization to moz.configure. r=gps
aclocal.m4
build/autoconf/python-virtualenv.m4
build/moz.configure/init.configure
build/moz.configure/old.configure
configure.py
js/src/aclocal.m4
js/src/old-configure.in
old-configure.in
--- a/aclocal.m4
+++ b/aclocal.m4
@@ -21,17 +21,16 @@ builtin(include, build/autoconf/mozcommo
 builtin(include, build/autoconf/lto.m4)dnl
 builtin(include, build/autoconf/frameptr.m4)dnl
 builtin(include, build/autoconf/compiler-opts.m4)dnl
 builtin(include, build/autoconf/expandlibs.m4)dnl
 builtin(include, build/autoconf/arch.m4)dnl
 builtin(include, build/autoconf/android.m4)dnl
 builtin(include, build/autoconf/zlib.m4)dnl
 builtin(include, build/autoconf/linux.m4)dnl
-builtin(include, build/autoconf/python-virtualenv.m4)dnl
 builtin(include, build/autoconf/winsdk.m4)dnl
 builtin(include, build/autoconf/icu.m4)dnl
 builtin(include, build/autoconf/ffi.m4)dnl
 builtin(include, build/autoconf/clang-plugin.m4)dnl
 builtin(include, build/autoconf/alloc.m4)dnl
 builtin(include, build/autoconf/ios.m4)dnl
 builtin(include, build/autoconf/jemalloc.m4)dnl
 builtin(include, build/autoconf/rust.m4)dnl
deleted file mode 100644
--- a/build/autoconf/python-virtualenv.m4
+++ /dev/null
@@ -1,85 +0,0 @@
-dnl This Source Code Form is subject to the terms of the Mozilla Public
-dnl License, v. 2.0. If a copy of the MPL was not distributed with this
-dnl file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-AC_DEFUN([MOZ_PYTHON],
-[
-
-dnl We honor the Python path defined in an environment variable. This is used
-dnl to pass the virtualenv's Python from the main configure to SpiderMonkey's
-dnl configure, for example.
-if test -z "$PYTHON"; then
-  MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python)
-  if test -z "$PYTHON"; then
-      AC_MSG_ERROR([python was not found in \$PATH])
-  fi
-else
-  AC_MSG_RESULT([Using Python from environment variable \$PYTHON])
-fi
-
-_virtualenv_topsrcdir=
-_virtualenv_populate_path=
-
-dnl If this is a mozilla-central, we'll find the virtualenv in the top
-dnl source directory. If this is a SpiderMonkey build, we assume we're at
-dnl js/src and try to find the virtualenv from the mozilla-central root.
-for base in $MOZILLA_CENTRAL_PATH $_topsrcdir $_topsrcdir/../..; do
-  possible=$base/python/mozbuild/mozbuild/virtualenv.py
-
-  if test -e $possible; then
-    _virtualenv_topsrcdir=$base
-    _virtualenv_populate_path=$possible
-    break
-  fi
-done
-
-if test -z $_virtualenv_populate_path; then
-    AC_MSG_ERROR([Unable to find Virtualenv population script. In order
-to build, you will need mozilla-central's virtualenv.
-
-If you are building from a mozilla-central checkout, you should never see this
-message. If you are building from a source archive, the source archive was
-likely not created properly (it is missing the virtualenv files).
-
-If you have a copy of mozilla-central available, define the
-MOZILLA_CENTRAL_PATH environment variable to the top source directory of
-mozilla-central and relaunch configure.])
-
-fi
-
-if test -z $DONT_POPULATE_VIRTUALENV; then
-  AC_MSG_RESULT([Creating Python environment])
-  dnl This verifies our Python version is sane and ensures the Python
-  dnl virtualenv is present and up to date. It sanitizes the environment
-  dnl for us, so we don't need to clean anything out.
-  $PYTHON $_virtualenv_populate_path \
-    $_virtualenv_topsrcdir $MOZ_BUILD_ROOT $MOZ_BUILD_ROOT/_virtualenv \
-    $_virtualenv_topsrcdir/build/virtualenv_packages.txt || exit 1
-
-  case "$host_os" in
-  mingw*)
-    PYTHON=`cd $MOZ_BUILD_ROOT && pwd -W`/_virtualenv/Scripts/python.exe
-    ;;
-  *)
-    PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
-    ;;
-  esac
-fi
-
-AC_SUBST(PYTHON)
-
-AC_MSG_CHECKING([Python environment is Mozilla virtualenv])
-$PYTHON -c "import mozbuild.base"
-if test "$?" != 0; then
-    AC_MSG_ERROR([Python environment does not appear to be sane.])
-fi
-AC_MSG_RESULT([yes])
-
-PYTHON_SITE_PACKAGES=`$PYTHON -c "import distutils.sysconfig; print distutils.sysconfig.get_python_lib()"`
-if test -z "$PYTHON_SITE_PACKAGES"; then
-    AC_MSG_ERROR([Could not determine python site packages directory.])
-fi
-AC_SUBST([PYTHON_SITE_PACKAGES])
-
-])
-
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -83,16 +83,94 @@ def mozconfig(current_project, mozconfig
     current_project = current_project[0] if current_project else None
     mozconfig = mozconfig[0] if mozconfig else None
     mozconfig = loader.find_mozconfig(env={'MOZCONFIG': mozconfig})
     mozconfig = loader.read_mozconfig(mozconfig, moz_build_app=current_project)
 
     return mozconfig
 
 
+option(env='PYTHON', nargs=1, help='Python interpreter')
+
+# Setup python virtualenv
+# ==============================================================
+@depends('PYTHON', check_build_environment, mozconfig)
+@advanced
+def virtualenv_python(env_python, build_env, mozconfig):
+    import os
+    import sys
+    import subprocess
+    from mozbuild.virtualenv import (
+	VirtualenvManager,
+        verify_python_version,
+    )
+
+    python = env_python[0] if env_python else None
+
+    # Ideally we'd rely on the mozconfig injection from mozconfig_options,
+    # but we'd rather avoid the verbosity when we need to reexecute with
+    # a different python.
+    if mozconfig['path']:
+        if 'PYTHON' in mozconfig['env']['added']:
+            python = mozconfig['env']['added']['PYTHON']
+        elif 'PYTHON' in mozconfig['env']['modified']:
+            python = mozconfig['env']['modified']['PYTHON'][1]
+        elif 'PYTHON' in mozconfig['vars']['added']:
+            python = mozconfig['vars']['added']['PYTHON']
+        elif 'PYTHON' in mozconfig['vars']['modified']:
+            python = mozconfig['vars']['modified']['PYTHON'][1]
+
+    verify_python_version(sys.stderr)
+    topsrcdir, topobjdir = build_env['TOPSRCDIR'], build_env['TOPOBJDIR']
+    if topobjdir.endswith('/js/src'):
+        topobjdir = topobjdir[:-7]
+
+    manager = VirtualenvManager(
+        topsrcdir, topobjdir,
+        os.path.join(topobjdir, '_virtualenv'), sys.stdout,
+        os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
+
+    if python:
+        # If we're not in the virtualenv, we need the which module for
+        # find_program.
+        if normsep(sys.executable) != normsep(manager.python_path):
+            sys.path.append(os.path.join(topsrcdir, 'python', 'which'))
+        found_python = find_program(python)
+        if not found_python:
+            error('The PYTHON environment variable does not contain '
+                  'a valid path. Cannot find %s' % python)
+        python = found_python
+    else:
+        python = sys.executable
+
+    if not manager.up_to_date(python):
+        warn('Creating Python environment')
+        manager.build(python)
+
+    python = normsep(manager.python_path)
+
+    if python != normsep(sys.executable):
+        warn('Reexecuting in the virtualenv')
+        if env_python:
+            del os.environ['PYTHON']
+        # One would prefer to use os.execl, but that's completely borked on
+        # Windows.
+        sys.exit(subprocess.call([python] + sys.argv))
+
+    # We are now in the virtualenv
+    import distutils.sysconfig
+    if not distutils.sysconfig.get_python_lib():
+        error('Could not determine python site packages directory')
+
+    set_config('PYTHON', python)
+    return python
+
+
+# Inject mozconfig options
+# ==============================================================
 @template
 @advanced
 def command_line_helper():
     # This escapes the sandbox. Don't copy this. This is only here because
     # it is a one off and because the required functionality doesn't need
     # to be exposed for other usecases.
     return depends.__self__._helper
 
--- a/build/moz.configure/old.configure
+++ b/build/moz.configure/old.configure
@@ -54,19 +54,21 @@ def autoconf(mozconfig, autoconf):
         error('Could not find autoconf 2.13')
 
     set_config('AUTOCONF', autoconf)
     return autoconf
 
 
 option(env='OLD_CONFIGURE', nargs=1, help='Path to the old configure script')
 
-@depends('OLD_CONFIGURE', mozconfig, autoconf, check_build_environment, shell)
+@depends('OLD_CONFIGURE', mozconfig, autoconf, check_build_environment, shell,
+         virtualenv_python)
 @advanced
-def prepare_configure(old_configure, mozconfig, autoconf, build_env, shell):
+def prepare_configure(old_configure, mozconfig, autoconf, build_env, shell,
+                      python):
     import glob
     import itertools
     import subprocess
     import sys
     # Import getmtime without overwriting the sandbox os.path.
     from os.path import getmtime
 
     from mozbuild.shellutil import quote
@@ -116,16 +118,18 @@ def prepare_configure(old_configure, moz
             for key, value in mozconfig['vars']['added'].items():
                 print("%s=%s" % (key, quote(value)), file=out)
             for key, (old, value) in mozconfig['vars']['modified'].items():
                 print("%s=%s" % (key, quote(value)), file=out)
             for t in ('env', 'vars'):
                 for key in mozconfig[t]['removed'].keys():
                     print("unset %s" % key, file=out)
 
+        print('PYTHON=%s' % quote(python), file=out)
+
     return cmd
 
 
 @template
 def old_configure_options(*options):
     for opt in options:
         option(opt, nargs='*', help='Help missing for old configure options')
 
--- a/configure.py
+++ b/configure.py
@@ -7,17 +7,16 @@ from __future__ import print_function, u
 import codecs
 import json
 import os
 import subprocess
 import sys
 
 
 base_dir = os.path.abspath(os.path.dirname(__file__))
-sys.path.append(os.path.join(base_dir, 'python', 'which'))
 sys.path.append(os.path.join(base_dir, 'python', 'mozbuild'))
 from mozbuild.configure import ConfigureSandbox
 
 
 def main(argv):
     config = {}
     sandbox = ConfigureSandbox(config, os.environ, argv)
     sandbox.run(os.path.join(os.path.dirname(__file__), 'moz.configure'))
--- a/js/src/aclocal.m4
+++ b/js/src/aclocal.m4
@@ -20,17 +20,16 @@ builtin(include, ../../build/autoconf/mo
 builtin(include, ../../build/autoconf/lto.m4)dnl
 builtin(include, ../../build/autoconf/frameptr.m4)dnl
 builtin(include, ../../build/autoconf/compiler-opts.m4)dnl
 builtin(include, ../../build/autoconf/expandlibs.m4)dnl
 builtin(include, ../../build/autoconf/arch.m4)dnl
 builtin(include, ../../build/autoconf/android.m4)dnl
 builtin(include, ../../build/autoconf/zlib.m4)dnl
 builtin(include, ../../build/autoconf/linux.m4)dnl
-builtin(include, ../../build/autoconf/python-virtualenv.m4)dnl
 builtin(include, ../../build/autoconf/winsdk.m4)dnl
 builtin(include, ../../build/autoconf/icu.m4)dnl
 builtin(include, ../../build/autoconf/ffi.m4)dnl
 builtin(include, ../../build/autoconf/clang-plugin.m4)dnl
 builtin(include, ../../build/autoconf/alloc.m4)dnl
 builtin(include, ../../build/autoconf/jemalloc.m4)dnl
 builtin(include, ../../build/autoconf/ios.m4)dnl
 builtin(include, ../../build/autoconf/rust.m4)dnl
@@ -40,8 +39,13 @@ define([AC_INIT_PREPARE],
 [if test -z "$srcdir"; then
   srcdir=`dirname "[$]0"`
 fi
 srcdir="$srcdir/../.."
 __MOZ_AC_INIT_PREPARE($1)
 ])
 
 MOZ_PROG_CHECKMSYS()
+dnl This won't actually read the mozconfig, but data that configure.py
+dnl will have placed for us to read. Configure.py takes care of not reading
+dnl the mozconfig where appropriate but can still give us some variables
+dnl to read.
+MOZ_READ_MOZCONFIG(.)
--- a/js/src/old-configure.in
+++ b/js/src/old-configure.in
@@ -522,18 +522,16 @@ AC_MSG_CHECKING([for full perl installat
 _perl_res=$?
 if test "$_perl_res" != 0; then
     AC_MSG_RESULT([no])
     AC_MSG_ERROR([Cannot find Config.pm or \$Config{archlib}.  A full perl installation is required.])
 else
     AC_MSG_RESULT([yes])
 fi
 
-MOZ_PYTHON
-
 if test -z "$COMPILE_ENVIRONMENT"; then
     NSINSTALL_BIN='$(PYTHON) $(topsrcdir)/config/nsinstall.py'
 fi
 AC_SUBST(NSINSTALL_BIN)
 
 MOZ_PATH_PROG(DOXYGEN, doxygen, :)
 MOZ_PATH_PROG(XARGS, xargs)
 if test -z "$XARGS" -o "$XARGS" = ":"; then
--- a/old-configure.in
+++ b/old-configure.in
@@ -94,18 +94,16 @@ MOZ_USE_PTHREADS=
 dnl Do not allow objdir == srcdir builds.
 dnl ==============================================================
 _topsrcdir=`cd \`dirname $0\`; pwd -W 2>/dev/null || pwd -P`
 _objdir=`pwd -P`
 
 MOZ_BUILD_ROOT=`pwd -W 2>/dev/null || pwd -P`
 DIST="$MOZ_BUILD_ROOT/dist"
 
-MOZ_PYTHON
-
 MOZ_DEFAULT_COMPILER
 
 COMPILE_ENVIRONMENT=1
 MOZ_ARG_DISABLE_BOOL(compile-environment,
 [  --disable-compile-environment
                           Disable compiler/library checks.],
     COMPILE_ENVIRONMENT= )
 AC_SUBST(COMPILE_ENVIRONMENT)
@@ -9101,18 +9099,16 @@ fi
 if test -n "$ZLIB_IN_MOZGLUE"; then
    MOZ_ZLIB_LIBS=
 fi
 export MOZ_NATIVE_ZLIB
 export MOZ_ZLIB_CFLAGS
 export MOZ_ZLIB_LIBS
 export MOZ_APP_NAME
 export MOZ_APP_REMOTINGNAME
-export DONT_POPULATE_VIRTUALENV=1
-export PYTHON
 export RUSTC
 export MOZILLA_CENTRAL_PATH=$_topsrcdir
 export STLPORT_CPPFLAGS
 export STLPORT_LIBS
 export JS_STANDALONE=no
 export DIST
 export MOZ_LINKER
 export ZLIB_IN_MOZGLUE