Bug 1253502 - Move python virtualenv initialization to moz.configure. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 04 Mar 2016 14:31:08 +0900
changeset 287136 33859b1a30fa988aed77d3260daee09ade2c938c
parent 287135 2439b4a424353eb0b3f97ca56717ac317790f24f
child 287137 addd5fc00284182a0ce28d302535266d7ef6faea
push id73071
push usermh@glandium.org
push dateTue, 08 Mar 2016 06:43:42 +0000
treeherdermozilla-inbound@addd5fc00284 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1253502
milestone47.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1253502 - Move python virtualenv initialization to moz.configure. r=gps
aclocal.m4
build/autoconf/python-virtualenv.m4
build/moz.configure/init.configure
build/moz.configure/old.configure
configure.py
js/src/aclocal.m4
js/src/old-configure.in
old-configure.in
--- a/aclocal.m4
+++ b/aclocal.m4
@@ -21,17 +21,16 @@ builtin(include, build/autoconf/mozcommo
 builtin(include, build/autoconf/lto.m4)dnl
 builtin(include, build/autoconf/frameptr.m4)dnl
 builtin(include, build/autoconf/compiler-opts.m4)dnl
 builtin(include, build/autoconf/expandlibs.m4)dnl
 builtin(include, build/autoconf/arch.m4)dnl
 builtin(include, build/autoconf/android.m4)dnl
 builtin(include, build/autoconf/zlib.m4)dnl
 builtin(include, build/autoconf/linux.m4)dnl
-builtin(include, build/autoconf/python-virtualenv.m4)dnl
 builtin(include, build/autoconf/winsdk.m4)dnl
 builtin(include, build/autoconf/icu.m4)dnl
 builtin(include, build/autoconf/ffi.m4)dnl
 builtin(include, build/autoconf/clang-plugin.m4)dnl
 builtin(include, build/autoconf/alloc.m4)dnl
 builtin(include, build/autoconf/ios.m4)dnl
 builtin(include, build/autoconf/jemalloc.m4)dnl
 builtin(include, build/autoconf/rust.m4)dnl
deleted file mode 100644
--- a/build/autoconf/python-virtualenv.m4
+++ /dev/null
@@ -1,85 +0,0 @@
-dnl This Source Code Form is subject to the terms of the Mozilla Public
-dnl License, v. 2.0. If a copy of the MPL was not distributed with this
-dnl file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-AC_DEFUN([MOZ_PYTHON],
-[
-
-dnl We honor the Python path defined in an environment variable. This is used
-dnl to pass the virtualenv's Python from the main configure to SpiderMonkey's
-dnl configure, for example.
-if test -z "$PYTHON"; then
-  MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python)
-  if test -z "$PYTHON"; then
-      AC_MSG_ERROR([python was not found in \$PATH])
-  fi
-else
-  AC_MSG_RESULT([Using Python from environment variable \$PYTHON])
-fi
-
-_virtualenv_topsrcdir=
-_virtualenv_populate_path=
-
-dnl If this is a mozilla-central, we'll find the virtualenv in the top
-dnl source directory. If this is a SpiderMonkey build, we assume we're at
-dnl js/src and try to find the virtualenv from the mozilla-central root.
-for base in $MOZILLA_CENTRAL_PATH $_topsrcdir $_topsrcdir/../..; do
-  possible=$base/python/mozbuild/mozbuild/virtualenv.py
-
-  if test -e $possible; then
-    _virtualenv_topsrcdir=$base
-    _virtualenv_populate_path=$possible
-    break
-  fi
-done
-
-if test -z $_virtualenv_populate_path; then
-    AC_MSG_ERROR([Unable to find Virtualenv population script. In order
-to build, you will need mozilla-central's virtualenv.
-
-If you are building from a mozilla-central checkout, you should never see this
-message. If you are building from a source archive, the source archive was
-likely not created properly (it is missing the virtualenv files).
-
-If you have a copy of mozilla-central available, define the
-MOZILLA_CENTRAL_PATH environment variable to the top source directory of
-mozilla-central and relaunch configure.])
-
-fi
-
-if test -z $DONT_POPULATE_VIRTUALENV; then
-  AC_MSG_RESULT([Creating Python environment])
-  dnl This verifies our Python version is sane and ensures the Python
-  dnl virtualenv is present and up to date. It sanitizes the environment
-  dnl for us, so we don't need to clean anything out.
-  $PYTHON $_virtualenv_populate_path \
-    $_virtualenv_topsrcdir $MOZ_BUILD_ROOT $MOZ_BUILD_ROOT/_virtualenv \
-    $_virtualenv_topsrcdir/build/virtualenv_packages.txt || exit 1
-
-  case "$host_os" in
-  mingw*)
-    PYTHON=`cd $MOZ_BUILD_ROOT && pwd -W`/_virtualenv/Scripts/python.exe
-    ;;
-  *)
-    PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
-    ;;
-  esac
-fi
-
-AC_SUBST(PYTHON)
-
-AC_MSG_CHECKING([Python environment is Mozilla virtualenv])
-$PYTHON -c "import mozbuild.base"
-if test "$?" != 0; then
-    AC_MSG_ERROR([Python environment does not appear to be sane.])
-fi
-AC_MSG_RESULT([yes])
-
-PYTHON_SITE_PACKAGES=`$PYTHON -c "import distutils.sysconfig; print distutils.sysconfig.get_python_lib()"`
-if test -z "$PYTHON_SITE_PACKAGES"; then
-    AC_MSG_ERROR([Could not determine python site packages directory.])
-fi
-AC_SUBST([PYTHON_SITE_PACKAGES])
-
-])
-
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -83,16 +83,94 @@ def mozconfig(current_project, mozconfig
     current_project = current_project[0] if current_project else None
     mozconfig = mozconfig[0] if mozconfig else None
     mozconfig = loader.find_mozconfig(env={'MOZCONFIG': mozconfig})
     mozconfig = loader.read_mozconfig(mozconfig, moz_build_app=current_project)
 
     return mozconfig
 
 
+option(env='PYTHON', nargs=1, help='Python interpreter')
+
+# Setup python virtualenv
+# ==============================================================
+@depends('PYTHON', check_build_environment, mozconfig)
+@advanced
+def virtualenv_python(env_python, build_env, mozconfig):
+    import os
+    import sys
+    import subprocess
+    from mozbuild.virtualenv import (
+	VirtualenvManager,
+        verify_python_version,
+    )
+
+    python = env_python[0] if env_python else None
+
+    # Ideally we'd rely on the mozconfig injection from mozconfig_options,
+    # but we'd rather avoid the verbosity when we need to reexecute with
+    # a different python.
+    if mozconfig['path']:
+        if 'PYTHON' in mozconfig['env']['added']:
+            python = mozconfig['env']['added']['PYTHON']
+        elif 'PYTHON' in mozconfig['env']['modified']:
+            python = mozconfig['env']['modified']['PYTHON'][1]
+        elif 'PYTHON' in mozconfig['vars']['added']:
+            python = mozconfig['vars']['added']['PYTHON']
+        elif 'PYTHON' in mozconfig['vars']['modified']:
+            python = mozconfig['vars']['modified']['PYTHON'][1]
+
+    verify_python_version(sys.stderr)
+    topsrcdir, topobjdir = build_env['TOPSRCDIR'], build_env['TOPOBJDIR']
+    if topobjdir.endswith('/js/src'):
+        topobjdir = topobjdir[:-7]
+
+    manager = VirtualenvManager(
+        topsrcdir, topobjdir,
+        os.path.join(topobjdir, '_virtualenv'), sys.stdout,
+        os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
+
+    if python:
+        # If we're not in the virtualenv, we need the which module for
+        # find_program.
+        if normsep(sys.executable) != normsep(manager.python_path):
+            sys.path.append(os.path.join(topsrcdir, 'python', 'which'))
+        found_python = find_program(python)
+        if not found_python:
+            error('The PYTHON environment variable does not contain '
+                  'a valid path. Cannot find %s' % python)
+        python = found_python
+    else:
+        python = sys.executable
+
+    if not manager.up_to_date(python):
+        warn('Creating Python environment')
+        manager.build(python)
+
+    python = normsep(manager.python_path)
+
+    if python != normsep(sys.executable):
+        warn('Reexecuting in the virtualenv')
+        if env_python:
+            del os.environ['PYTHON']
+        # One would prefer to use os.execl, but that's completely borked on
+        # Windows.
+        sys.exit(subprocess.call([python] + sys.argv))
+
+    # We are now in the virtualenv
+    import distutils.sysconfig
+    if not distutils.sysconfig.get_python_lib():
+        error('Could not determine python site packages directory')
+
+    set_config('PYTHON', python)
+    return python
+
+
+# Inject mozconfig options
+# ==============================================================
 @template
 @advanced
 def command_line_helper():
     # This escapes the sandbox. Don't copy this. This is only here because
     # it is a one off and because the required functionality doesn't need
     # to be exposed for other usecases.
     return depends.__self__._helper
 
--- a/build/moz.configure/old.configure
+++ b/build/moz.configure/old.configure
@@ -54,19 +54,21 @@ def autoconf(mozconfig, autoconf):
         error('Could not find autoconf 2.13')
 
     set_config('AUTOCONF', autoconf)
     return autoconf
 
 
 option(env='OLD_CONFIGURE', nargs=1, help='Path to the old configure script')
 
-@depends('OLD_CONFIGURE', mozconfig, autoconf, check_build_environment, shell)
+@depends('OLD_CONFIGURE', mozconfig, autoconf, check_build_environment, shell,
+         virtualenv_python)
 @advanced
-def prepare_configure(old_configure, mozconfig, autoconf, build_env, shell):
+def prepare_configure(old_configure, mozconfig, autoconf, build_env, shell,
+                      python):
     import glob
     import itertools
     import subprocess
     import sys
     # Import getmtime without overwriting the sandbox os.path.
     from os.path import getmtime
 
     from mozbuild.shellutil import quote
@@ -116,16 +118,18 @@ def prepare_configure(old_configure, moz
             for key, value in mozconfig['vars']['added'].items():
                 print("%s=%s" % (key, quote(value)), file=out)
             for key, (old, value) in mozconfig['vars']['modified'].items():
                 print("%s=%s" % (key, quote(value)), file=out)
             for t in ('env', 'vars'):
                 for key in mozconfig[t]['removed'].keys():
                     print("unset %s" % key, file=out)
 
+        print('PYTHON=%s' % quote(python), file=out)
+
     return cmd
 
 
 @template
 def old_configure_options(*options):
     for opt in options:
         option(opt, nargs='*', help='Help missing for old configure options')
 
--- a/configure.py
+++ b/configure.py
@@ -7,17 +7,16 @@ from __future__ import print_function, u
 import codecs
 import json
 import os
 import subprocess
 import sys
 
 
 base_dir = os.path.abspath(os.path.dirname(__file__))
-sys.path.append(os.path.join(base_dir, 'python', 'which'))
 sys.path.append(os.path.join(base_dir, 'python', 'mozbuild'))
 from mozbuild.configure import ConfigureSandbox
 
 
 def main(argv):
     config = {}
     sandbox = ConfigureSandbox(config, os.environ, argv)
     sandbox.run(os.path.join(os.path.dirname(__file__), 'moz.configure'))
--- a/js/src/aclocal.m4
+++ b/js/src/aclocal.m4
@@ -20,17 +20,16 @@ builtin(include, ../../build/autoconf/mo
 builtin(include, ../../build/autoconf/lto.m4)dnl
 builtin(include, ../../build/autoconf/frameptr.m4)dnl
 builtin(include, ../../build/autoconf/compiler-opts.m4)dnl
 builtin(include, ../../build/autoconf/expandlibs.m4)dnl
 builtin(include, ../../build/autoconf/arch.m4)dnl
 builtin(include, ../../build/autoconf/android.m4)dnl
 builtin(include, ../../build/autoconf/zlib.m4)dnl
 builtin(include, ../../build/autoconf/linux.m4)dnl
-builtin(include, ../../build/autoconf/python-virtualenv.m4)dnl
 builtin(include, ../../build/autoconf/winsdk.m4)dnl
 builtin(include, ../../build/autoconf/icu.m4)dnl
 builtin(include, ../../build/autoconf/ffi.m4)dnl
 builtin(include, ../../build/autoconf/clang-plugin.m4)dnl
 builtin(include, ../../build/autoconf/alloc.m4)dnl
 builtin(include, ../../build/autoconf/jemalloc.m4)dnl
 builtin(include, ../../build/autoconf/ios.m4)dnl
 builtin(include, ../../build/autoconf/rust.m4)dnl
@@ -40,8 +39,13 @@ define([AC_INIT_PREPARE],
 [if test -z "$srcdir"; then
   srcdir=`dirname "[$]0"`
 fi
 srcdir="$srcdir/../.."
 __MOZ_AC_INIT_PREPARE($1)
 ])
 
 MOZ_PROG_CHECKMSYS()
+dnl This won't actually read the mozconfig, but data that configure.py
+dnl will have placed for us to read. Configure.py takes care of not reading
+dnl the mozconfig where appropriate but can still give us some variables
+dnl to read.
+MOZ_READ_MOZCONFIG(.)
--- a/js/src/old-configure.in
+++ b/js/src/old-configure.in
@@ -522,18 +522,16 @@ AC_MSG_CHECKING([for full perl installat
 _perl_res=$?
 if test "$_perl_res" != 0; then
     AC_MSG_RESULT([no])
     AC_MSG_ERROR([Cannot find Config.pm or \$Config{archlib}.  A full perl installation is required.])
 else
     AC_MSG_RESULT([yes])
 fi
 
-MOZ_PYTHON
-
 if test -z "$COMPILE_ENVIRONMENT"; then
     NSINSTALL_BIN='$(PYTHON) $(topsrcdir)/config/nsinstall.py'
 fi
 AC_SUBST(NSINSTALL_BIN)
 
 MOZ_PATH_PROG(DOXYGEN, doxygen, :)
 MOZ_PATH_PROG(XARGS, xargs)
 if test -z "$XARGS" -o "$XARGS" = ":"; then
--- a/old-configure.in
+++ b/old-configure.in
@@ -94,18 +94,16 @@ MOZ_USE_PTHREADS=
 dnl Do not allow objdir == srcdir builds.
 dnl ==============================================================
 _topsrcdir=`cd \`dirname $0\`; pwd -W 2>/dev/null || pwd -P`
 _objdir=`pwd -P`
 
 MOZ_BUILD_ROOT=`pwd -W 2>/dev/null || pwd -P`
 DIST="$MOZ_BUILD_ROOT/dist"
 
-MOZ_PYTHON
-
 MOZ_DEFAULT_COMPILER
 
 COMPILE_ENVIRONMENT=1
 MOZ_ARG_DISABLE_BOOL(compile-environment,
 [  --disable-compile-environment
                           Disable compiler/library checks.],
     COMPILE_ENVIRONMENT= )
 AC_SUBST(COMPILE_ENVIRONMENT)
@@ -9101,18 +9099,16 @@ fi
 if test -n "$ZLIB_IN_MOZGLUE"; then
    MOZ_ZLIB_LIBS=
 fi
 export MOZ_NATIVE_ZLIB
 export MOZ_ZLIB_CFLAGS
 export MOZ_ZLIB_LIBS
 export MOZ_APP_NAME
 export MOZ_APP_REMOTINGNAME
-export DONT_POPULATE_VIRTUALENV=1
-export PYTHON
 export RUSTC
 export MOZILLA_CENTRAL_PATH=$_topsrcdir
 export STLPORT_CPPFLAGS
 export STLPORT_LIBS
 export JS_STANDALONE=no
 export DIST
 export MOZ_LINKER
 export ZLIB_IN_MOZGLUE