Bug 1253502 - Move python virtualenv initialization to moz.configure
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 04 Mar 2016 14:31:08 +0900
changeset 708851 acdd1844f4ebb454fd13c1bb763ead8db91b5ad6
parent 708850 b467342bab60c66c3d2fc670eb00492a9469b132
child 708852 ab0ec86b8063bb4823fc38c07f7028be5074a915
push id112728
push usermh@glandium.org
push dateFri, 04 Mar 2016 06:30:03 +0000
treeherdertry@ab0ec86b8063 [default view] [failures only]
Bug 1253502 - Move python virtualenv initialization to moz.configure
--- a/aclocal.m4
+++ b/aclocal.m4
@@ -21,17 +21,16 @@ builtin(include, build/autoconf/mozcommo
 builtin(include, build/autoconf/lto.m4)dnl
 builtin(include, build/autoconf/frameptr.m4)dnl
 builtin(include, build/autoconf/compiler-opts.m4)dnl
 builtin(include, build/autoconf/expandlibs.m4)dnl
 builtin(include, build/autoconf/arch.m4)dnl
 builtin(include, build/autoconf/android.m4)dnl
 builtin(include, build/autoconf/zlib.m4)dnl
 builtin(include, build/autoconf/linux.m4)dnl
-builtin(include, build/autoconf/python-virtualenv.m4)dnl
 builtin(include, build/autoconf/winsdk.m4)dnl
 builtin(include, build/autoconf/icu.m4)dnl
 builtin(include, build/autoconf/ffi.m4)dnl
 builtin(include, build/autoconf/clang-plugin.m4)dnl
 builtin(include, build/autoconf/alloc.m4)dnl
 builtin(include, build/autoconf/ios.m4)dnl
 builtin(include, build/autoconf/jemalloc.m4)dnl
 builtin(include, build/autoconf/rust.m4)dnl
deleted file mode 100644
--- a/build/autoconf/python-virtualenv.m4
+++ /dev/null
@@ -1,85 +0,0 @@
-dnl This Source Code Form is subject to the terms of the Mozilla Public
-dnl License, v. 2.0. If a copy of the MPL was not distributed with this
-dnl file, You can obtain one at http://mozilla.org/MPL/2.0/.
-dnl We honor the Python path defined in an environment variable. This is used
-dnl to pass the virtualenv's Python from the main configure to SpiderMonkey's
-dnl configure, for example.
-if test -z "$PYTHON"; then
-  MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python)
-  if test -z "$PYTHON"; then
-      AC_MSG_ERROR([python was not found in \$PATH])
-  fi
-  AC_MSG_RESULT([Using Python from environment variable \$PYTHON])
-dnl If this is a mozilla-central, we'll find the virtualenv in the top
-dnl source directory. If this is a SpiderMonkey build, we assume we're at
-dnl js/src and try to find the virtualenv from the mozilla-central root.
-for base in $MOZILLA_CENTRAL_PATH $_topsrcdir $_topsrcdir/../..; do
-  possible=$base/python/mozbuild/mozbuild/virtualenv.py
-  if test -e $possible; then
-    _virtualenv_topsrcdir=$base
-    _virtualenv_populate_path=$possible
-    break
-  fi
-if test -z $_virtualenv_populate_path; then
-    AC_MSG_ERROR([Unable to find Virtualenv population script. In order
-to build, you will need mozilla-central's virtualenv.
-If you are building from a mozilla-central checkout, you should never see this
-message. If you are building from a source archive, the source archive was
-likely not created properly (it is missing the virtualenv files).
-If you have a copy of mozilla-central available, define the
-MOZILLA_CENTRAL_PATH environment variable to the top source directory of
-mozilla-central and relaunch configure.])
-  AC_MSG_RESULT([Creating Python environment])
-  dnl This verifies our Python version is sane and ensures the Python
-  dnl virtualenv is present and up to date. It sanitizes the environment
-  dnl for us, so we don't need to clean anything out.
-  $PYTHON $_virtualenv_populate_path \
-    $_virtualenv_topsrcdir $MOZ_BUILD_ROOT $MOZ_BUILD_ROOT/_virtualenv \
-    $_virtualenv_topsrcdir/build/virtualenv_packages.txt || exit 1
-  case "$host_os" in
-  mingw*)
-    PYTHON=`cd $MOZ_BUILD_ROOT && pwd -W`/_virtualenv/Scripts/python.exe
-    ;;
-  *)
-    PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
-    ;;
-  esac
-AC_MSG_CHECKING([Python environment is Mozilla virtualenv])
-$PYTHON -c "import mozbuild.base"
-if test "$?" != 0; then
-    AC_MSG_ERROR([Python environment does not appear to be sane.])
-PYTHON_SITE_PACKAGES=`$PYTHON -c "import distutils.sysconfig; print distutils.sysconfig.get_python_lib()"`
-if test -z "$PYTHON_SITE_PACKAGES"; then
-    AC_MSG_ERROR([Could not determine python site packages directory.])
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -81,16 +81,77 @@ def mozconfig(current_project, mozconfig
     current_project = current_project[0] if current_project else None
     mozconfig = mozconfig[0] if mozconfig else None
     mozconfig = loader.find_mozconfig(env={'MOZCONFIG': mozconfig})
     mozconfig = loader.read_mozconfig(mozconfig, moz_build_app=current_project)
     return mozconfig
+option(env='PYTHON', nargs=1, help='Python interpreter')
+# Setup python virtualenv
+# ==============================================================
+@depends('PYTHON', check_build_environment, '--help')
+def virtualenv_python(env_python, build_env, help):
+    # We don't want to require a virtualenv to display the help.
+    if help:
+        return
+    import os
+    import sys
+    from mozbuild.virtualenv import (
+	VirtualenvManager,
+        verify_python_version,
+    )
+    verify_python_version(sys.stderr)
+    topsrcdir, topobjdir = build_env['TOPSRCDIR'], build_env['TOPOBJDIR']
+    manager = VirtualenvManager(
+        topsrcdir, topobjdir,
+        os.path.join(topobjdir, '_virtualenv'), sys.stdout,
+        os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
+    if env_python:
+        # If we're not in the virtualenv, we need the which module for
+        # find_program.
+        if sys.executable != manager.python_path:
+            sys.path.append(os.path.join(topsrcdir, 'python', 'which'))
+        python = find_program(env_python[0])
+        if not python:
+            error('The PYTHON environment variable does not contain '
+                  'a valid path. Cannot find %s' % env_python[0])
+    else:
+        python = sys.executable
+    if not manager.up_to_date(python):
+        warn('Creating Python environment')
+        manager.build(python)
+    python = manager.python_path
+    if python != sys.executable:
+        warn('Reexecuting in the virtualenv')
+        if env_python:
+            del os.environ['PYTHON']
+        os.execl(python, python, *sys.argv)
+        error('Failed to reexecute in the virtualenv')
+    # We are now in the virtualenv
+    import distutils.sysconfig
+    if not distutils.sysconfig.get_python_lib():
+        error('Could not determine python site packages directory')
+    set_config('PYTHON', python)
+    return python
+# Inject mozconfig options
+# ==============================================================
 def command_line_helper():
     # This escapes the sandbox. Don't copy this. This is only here because
     # it is a one off and because the required functionality doesn't need
     # to be exposed for other usecases.
     return depends.__self__._helper
--- a/build/moz.configure/old.configure
+++ b/build/moz.configure/old.configure
@@ -54,19 +54,21 @@ def autoconf(mozconfig, autoconf):
         error('Could not find autoconf 2.13')
     set_config('AUTOCONF', autoconf)
     return autoconf
 option(env='OLD_CONFIGURE', nargs=1, help='Path to the old configure script')
-@depends('OLD_CONFIGURE', mozconfig, autoconf, check_build_environment, shell)
+@depends('OLD_CONFIGURE', mozconfig, autoconf, check_build_environment, shell,
+         virtualenv_python)
-def prepare_configure(old_configure, mozconfig, autoconf, build_env, shell):
+def prepare_configure(old_configure, mozconfig, autoconf, build_env, shell,
+                      python):
     import glob
     import itertools
     import os
     import subprocess
     import sys
     from mozbuild.shellutil import quote
@@ -114,16 +116,18 @@ def prepare_configure(old_configure, moz
             for key, value in mozconfig['vars']['added'].items():
                 print("%s=%s" % (key, quote(value)), file=out)
             for key, (old, value) in mozconfig['vars']['modified'].items():
                 print("%s=%s" % (key, quote(value)), file=out)
             for t in ('env', 'vars'):
                 for key in mozconfig[t]['removed'].keys():
                     print("unset %s" % key, file=out)
+        print('PYTHON=%s' % quote(python), file=out)
     return cmd
 def old_configure_options(*options):
     for opt in options:
         option(opt, nargs='*', help='Help missing for old configure options')
--- a/configure.py
+++ b/configure.py
@@ -7,17 +7,16 @@ from __future__ import print_function, u
 import codecs
 import json
 import os
 import subprocess
 import sys
 base_dir = os.path.abspath(os.path.dirname(__file__))
-sys.path.append(os.path.join(base_dir, 'python', 'which'))
 sys.path.append(os.path.join(base_dir, 'python', 'mozbuild'))
 from mozbuild.configure import ConfigureSandbox
 def main(argv):
     config = {}
     sandbox = ConfigureSandbox(config, os.environ, argv)
     sandbox.run(os.path.join(os.path.dirname(__file__), 'moz.configure'))
--- a/js/src/aclocal.m4
+++ b/js/src/aclocal.m4
@@ -20,17 +20,16 @@ builtin(include, ../../build/autoconf/mo
 builtin(include, ../../build/autoconf/lto.m4)dnl
 builtin(include, ../../build/autoconf/frameptr.m4)dnl
 builtin(include, ../../build/autoconf/compiler-opts.m4)dnl
 builtin(include, ../../build/autoconf/expandlibs.m4)dnl
 builtin(include, ../../build/autoconf/arch.m4)dnl
 builtin(include, ../../build/autoconf/android.m4)dnl
 builtin(include, ../../build/autoconf/zlib.m4)dnl
 builtin(include, ../../build/autoconf/linux.m4)dnl
-builtin(include, ../../build/autoconf/python-virtualenv.m4)dnl
 builtin(include, ../../build/autoconf/winsdk.m4)dnl
 builtin(include, ../../build/autoconf/icu.m4)dnl
 builtin(include, ../../build/autoconf/ffi.m4)dnl
 builtin(include, ../../build/autoconf/clang-plugin.m4)dnl
 builtin(include, ../../build/autoconf/alloc.m4)dnl
 builtin(include, ../../build/autoconf/jemalloc.m4)dnl
 builtin(include, ../../build/autoconf/ios.m4)dnl
 builtin(include, ../../build/autoconf/rust.m4)dnl
@@ -40,8 +39,13 @@ define([AC_INIT_PREPARE],
 [if test -z "$srcdir"; then
   srcdir=`dirname "[$]0"`
+dnl This won't actually read the mozconfig, but data that configure.py
+dnl will have placed for us to read. Configure.py takes care of not reading
+dnl the mozconfig where appropriate but can still give us some variables
+dnl to read.
--- a/js/src/old-configure.in
+++ b/js/src/old-configure.in
@@ -522,18 +522,16 @@ AC_MSG_CHECKING([for full perl installat
 if test "$_perl_res" != 0; then
     AC_MSG_ERROR([Cannot find Config.pm or \$Config{archlib}.  A full perl installation is required.])
 if test -z "$COMPILE_ENVIRONMENT"; then
     NSINSTALL_BIN='$(PYTHON) $(topsrcdir)/config/nsinstall.py'
 if test -z "$XARGS" -o "$XARGS" = ":"; then
--- a/old-configure.in
+++ b/old-configure.in
@@ -94,18 +94,16 @@ MOZ_USE_PTHREADS=
 dnl Do not allow objdir == srcdir builds.
 dnl ==============================================================
 _topsrcdir=`cd \`dirname $0\`; pwd -W 2>/dev/null || pwd -P`
 _objdir=`pwd -P`
 MOZ_BUILD_ROOT=`pwd -W 2>/dev/null || pwd -P`
 [  --disable-compile-environment
                           Disable compiler/library checks.],