Bug 1046533 - Completely wrap subconfigures. r=mshal
authorMike Hommey <mh+mozilla@glandium.org>
Sat, 02 Aug 2014 08:02:30 +0900
changeset 219186 112cf093670bc0e955da224b2738a80a68656293
parent 219185 100721f9718f94363f646a7b3d7b7d13f220326b
child 219187 5d3878325b7ea9bd3492b3c8e9d86a1be4e18bb9
push id3979
push userraliiev@mozilla.com
push dateMon, 13 Oct 2014 16:35:44 +0000
treeherdermozilla-beta@30f2cc610691 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmshal
bugs1046533, 903369
milestone34.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1046533 - Completely wrap subconfigures. r=mshal While bug 903369 added some kind of wrapping, msys mangling on Windows made it hard to make the python wrapper invoke subconfigures itself. This change overcomes this, allowing to run subconfigures entirely independently of the main configure if necessary, or to do more fancy checks without having to resort to m4 and shell.
build/autoconf/hooks.m4
build/clang-plugin/configure
build/subconfigure.py
build/win32/dumpenv4python.pl
--- a/build/autoconf/hooks.m4
+++ b/build/autoconf/hooks.m4
@@ -12,57 +12,52 @@ changequote([, ])dnl
 
 dnl Wrap AC_INIT_PREPARE to add the above trap.
 define([_MOZ_AC_INIT_PREPARE], defn([AC_INIT_PREPARE]))
 define([AC_INIT_PREPARE],
 [_MOZ_AC_INIT_PREPARE($1)
 MOZ_CONFIG_LOG_TRAP
 ])
 
-dnl Disable the trap when running sub-configures.
-define(GEN_MOZ_AC_OUTPUT_SUBDIRS, [
-define([_MOZ_AC_OUTPUT_SUBDIRS], [
-patsubst($@, [$srcdir/$ac_config_dir], [$srcdir/$moz_config_srcdir])
-])
-])
-GEN_MOZ_AC_OUTPUT_SUBDIRS(defn([AC_OUTPUT_SUBDIRS]))
-
 define([AC_OUTPUT_SUBDIRS],
-[trap '' EXIT
-for moz_config_dir in $1; do
+[for moz_config_dir in $1; do
+  _CONFIG_SHELL=${CONFIG_SHELL-/bin/sh}
   case "$moz_config_dir" in
   *:*)
-    moz_config_srcdir=$(echo $moz_config_dir | awk -F: '{print [$]1}')
-    moz_config_dir=$(echo $moz_config_dir | awk -F: '{print [$]2}')
+    objdir=$(echo $moz_config_dir | awk -F: '{print [$]2}')
     ;;
   *)
-    moz_config_srcdir=$moz_config_dir
+    objdir=$moz_config_dir
     ;;
   esac
-  _CONFIG_SHELL=${CONFIG_SHELL-/bin/sh}
+
+  dumpenv="true | "
   case "$host" in
   *-mingw*)
     _CONFIG_SHELL=$(cd $(dirname $_CONFIG_SHELL); pwd -W)/$(basename $_CONFIG_SHELL)
     if test ! -e "$_CONFIG_SHELL" -a -e "${_CONFIG_SHELL}.exe"; then
         _CONFIG_SHELL="${_CONFIG_SHELL}.exe"
     fi
+    dnl Yes, this is horrible. But since msys doesn't preserve environment
+    dnl variables and command line arguments as they are when transitioning
+    dnl from msys (this script) to python (below), we have to resort to hacks,
+    dnl storing the environment and command line arguments from a msys process
+    dnl (perl), and reading it from python.
+    dumpenv="$PERL $srcdir/build/win32/dumpenv4python.pl $ac_configure_args | "
     ;;
   esac
 
-  if test -d "$moz_config_dir"; then
-    (cd "$moz_config_dir"; eval $PYTHON $_topsrcdir/build/subconfigure.py dump "$_CONFIG_SHELL" $ac_configure_args)
-  else
-    mkdir -p "$moz_config_dir"
+  eval $dumpenv $PYTHON $_topsrcdir/build/subconfigure.py --prepare "$srcdir" "$moz_config_dir" "$_CONFIG_SHELL" $ac_configure_args ifelse($2,,,--cache-file="$2")
+
+  dnl Execute subconfigure, unless --no-recursion was passed to configure.
+  if test "$no_recursion" != yes; then
+    trap '' EXIT
+    if ! $PYTHON $_topsrcdir/build/subconfigure.py "$objdir"; then
+        exit 1
+    fi
+    MOZ_CONFIG_LOG_TRAP
   fi
-  _save_cache_file="$cache_file"
-  ifelse($2,,cache_file="$moz_config_dir/config.cache",cache_file="$2")
-  cache_file="$(cd $(dirname "$cache_file"); pwd -W 2>/dev/null || pwd)/$(basename "$cache_file")"
-  _MOZ_AC_OUTPUT_SUBDIRS($moz_config_dir)
-  cache_file="$_save_cache_file"
-  (cd "$moz_config_dir"; $PYTHON $_topsrcdir/build/subconfigure.py adjust $ac_sub_configure)
 done
-
-MOZ_CONFIG_LOG_TRAP
 ])
 
 dnl Print error messages in config.log as well as stderr
 define([AC_MSG_ERROR],
 [{ echo "configure: error: $1" 1>&2; echo "configure: error: $1" 1>&5; exit 1; }])
--- a/build/clang-plugin/configure
+++ b/build/clang-plugin/configure
@@ -1,14 +1,14 @@
 #!/bin/sh
 
 PLATFORM=`uname`
 
 # Default srcdir to this directory
-srcdir=
+srcdir=$(dirname $0)
 
 for option; do
   case "$option" in
   -*=*) optarg=`echo "$option" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
   *) optarg= ;;
   esac
 
   case "$option" in
--- a/build/subconfigure.py
+++ b/build/subconfigure.py
@@ -1,22 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # This script is used to capture the content of config.status-generated
 # files and subsequently restore their timestamp if they haven't changed.
 
 import argparse
+import errno
 import os
 import re
 import subprocess
 import sys
 import pickle
 
+import mozpack.path as mozpath
+
 class File(object):
     def __init__(self, path):
         self._path = path
         self._content = open(path, 'rb').read()
         stat = os.stat(path)
         self._times = (stat.st_atime, stat.st_mtime)
 
     @property
@@ -56,113 +59,218 @@ PRECIOUS_VARS = set([
 
 
 # Autoconf, in some of the sub-configures used in the tree, likes to error
 # out when "precious" variables change in value. The solution it gives to
 # straighten things is to either run make distclean or remove config.cache.
 # There's no reason not to do the latter automatically instead of failing,
 # doing the cleanup (which, on buildbots means a full clobber), and
 # restarting from scratch.
-def maybe_clear_cache(args):
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--target', type=str)
-    parser.add_argument('--host', type=str)
-    parser.add_argument('--build', type=str)
-    args, others = parser.parse_known_args(args)
-    env = dict(os.environ)
+def maybe_clear_cache(data):
+    env = dict(data['env'])
     for kind in ('target', 'host', 'build'):
-        arg = getattr(args, kind)
+        arg = data[kind]
         if arg is not None:
             env['%s_alias' % kind] = arg
     # configure can take variables assignments in its arguments, and that
     # overrides whatever is in the environment.
-    for arg in others:
+    for arg in data['args']:
         if arg[:1] != '-' and '=' in arg:
             key, value = arg.split('=', 1)
             env[key] = value
 
     comment = re.compile(r'^\s+#')
     cache = {}
-    with open('config.cache') as f:
+    with open(data['cache-file']) as f:
         for line in f:
             if not comment.match(line) and '=' in line:
                 key, value = line.rstrip(os.linesep).split('=', 1)
                 # If the value is quoted, unquote it
                 if value[:1] == "'":
                     value = value[1:-1].replace("'\\''", "'")
                 cache[key] = value
     for precious in PRECIOUS_VARS:
         # If there is no entry at all for that precious variable, then
         # its value is not precious for that particular configure.
         if 'ac_cv_env_%s_set' % precious not in cache:
             continue
         is_set = cache.get('ac_cv_env_%s_set' % precious) == 'set'
         value = cache.get('ac_cv_env_%s_value' % precious) if is_set else None
         if value != env.get(precious):
-            print 'Removing config.cache because of %s value change from:' \
-                % precious
+            print 'Removing %s because of %s value change from:' \
+                % (data['cache-file'], precious)
             print '  %s' % (value if value is not None else 'undefined')
             print 'to:'
             print '  %s' % env.get(precious, 'undefined')
-            os.remove('config.cache')
+            os.remove(data['cache-file'])
             return
 
 
-def dump(dump_file, shell, args):
-    if os.path.exists('config.cache'):
-        maybe_clear_cache(args)
-    if not os.path.exists('config.status'):
-        if os.path.exists(dump_file):
-            os.remove(dump_file)
-        return
+def split_template(s):
+    """Given a "file:template" string, returns "file", "template". If the string
+    is of the form "file" (without a template), returns "file", "file.in"."""
+    if ':' in s:
+        return s.split(':', 1)
+    return s, '%s.in' % s
+
 
-    config_files = [File('config.status')]
+def get_config_files(data):
+    config_status = mozpath.join(data['objdir'], 'config.status')
+    if not os.path.exists(config_status):
+        return [], []
+
+    configure = mozpath.join(data['srcdir'], 'configure')
+    config_files = [(config_status, configure)]
+    command_files = []
 
     # Scan the config.status output for information about configuration files
     # it generates.
     config_status_output = subprocess.check_output(
-        [shell, '-c', './config.status --help'],
+        [data['shell'], '-c', '%s --help' % config_status],
         stderr=subprocess.STDOUT).splitlines()
     state = None
     for line in config_status_output:
         if line.startswith('Configuration') and line.endswith(':'):
-            state = 'config'
-        elif not line.startswith(' '):
+            if line.endswith('commands:'):
+                state = 'commands'
+            else:
+                state = 'config'
+        elif not line.strip():
             state = None
-        elif state == 'config':
-            for f in (couple.split(':')[0] for couple in line.split()):
-                if os.path.isfile(f):
-                    config_files.append(File(f))
+        elif state:
+            for f, t in (split_template(couple) for couple in line.split()):
+                f = mozpath.join(data['objdir'], f)
+                t = mozpath.join(data['srcdir'], t)
+                if state == 'commands':
+                    command_files.append(f)
+                else:
+                    config_files.append((f, t))
 
-    with open(dump_file, 'wb') as f:
-        pickle.dump(config_files, f)
+    return config_files, command_files
 
 
-def adjust(dump_file, configure):
-    if not os.path.exists(dump_file):
-        return
+def prepare(data_file, srcdir, objdir, shell, args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--target', type=str)
+    parser.add_argument('--host', type=str)
+    parser.add_argument('--build', type=str)
+    parser.add_argument('--cache-file', type=str)
+    # The --srcdir argument is simply ignored. It's a useless autoconf feature
+    # that we don't support well anyways. This makes it stripped from `others`
+    # and allows to skip setting it when calling the subconfigure (configure
+    # will take it from the configure path anyways).
+    parser.add_argument('--srcdir', type=str)
 
-    config_files = []
+    # Msys likes to break environment variables and command line arguments,
+    # so read those from stdin, as they are passed from the configure script
+    # when necessary (on windows).
+    # However, for some reason, $PATH is not handled like other environment
+    # variables, and msys remangles it even when giving it is already a msys
+    # $PATH. Fortunately, the mangling/demangling is just find for $PATH, so
+    # we can just take the value from the environment. Msys will convert it
+    # back properly when calling subconfigure.
+    input = sys.stdin.read()
+    if input:
+        data = {a: b for [a, b] in eval(input)}
+        environ = {a: b for a, b in data['env']}
+        environ['PATH'] = os.environ['PATH']
+        args = data['args']
+    else:
+        environ = os.environ
+
+    args, others = parser.parse_known_args(args)
+
+    data = {
+        'target': args.target,
+        'host': args.host,
+        'build': args.build,
+        'args': others,
+        'shell': shell,
+        'srcdir': srcdir,
+        'env': environ,
+    }
+
+    if args.cache_file:
+        data['cache-file'] = mozpath.normpath(mozpath.join(os.getcwd(),
+            args.cache_file))
+    else:
+        data['cache-file'] = mozpath.join(objdir, 'config.cache')
 
     try:
-        with open(dump_file, 'rb') as f:
-            config_files = pickle.load(f)
-    except Exception:
-        pass
+        os.makedirs(objdir)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+
+    with open(os.path.join(objdir, data_file), 'wb') as f:
+        pickle.dump(data, f)
+
+
+def run(data_file, objdir):
+    with open(os.path.join(objdir, data_file), 'rb') as f:
+        data = pickle.load(f)
+
+    data['objdir'] = objdir
+
+    cache_file = data['cache-file']
+    if os.path.exists(cache_file):
+        maybe_clear_cache(data)
 
-    for f in config_files:
+    config_files, command_files = get_config_files(data)
+    contents = []
+    for f, t in config_files:
+        contents.append(File(f))
+
+    # AC_CONFIG_COMMANDS actually only registers tags, not file names
+    # but most commands are tagged with the file name they create.
+    # However, a few don't, or are tagged with a directory name (and their
+    # command is just to create that directory)
+    for f in command_files:
+        if os.path.isfile(f):
+            contents.append(File(f))
+
+    configure = mozpath.join(data['srcdir'], 'configure')
+    command = [data['shell'], configure]
+    for kind in ('target', 'build', 'host'):
+        if data.get(kind) is not None:
+            command += ['--%s=%s' % (kind, data[kind])]
+    command += data['args']
+    command += ['--cache-file=%s' % cache_file]
+
+    print 'configuring in %s' % os.path.relpath(objdir, os.getcwd())
+    print 'running %s' % ' '.join(command)
+    sys.stdout.flush()
+    ret = subprocess.call(command, cwd=objdir, env=data['env'])
+
+    for f in contents:
         # Still touch config.status if configure is newer than its original
         # mtime.
-        if configure and os.path.basename(f.path) == 'config.status' and \
+        if os.path.basename(f.path) == 'config.status' and \
                 os.path.getmtime(configure) > f.mtime:
             continue
         f.update_time()
 
-    os.remove(dump_file)
+    return ret
 
 
-CONFIG_DUMP = 'config_files.pkl'
+CONFIGURE_DATA = 'configure.pkl'
+
+def main(args):
+    if args[0] != '--prepare':
+        if len(args) != 1:
+            raise Exception('Usage: %s relativeobjdir' % __file__)
+        return run(CONFIGURE_DATA, args[0])
+
+    topsrcdir = os.path.abspath(args[1])
+    subdir = args[2]
+    # subdir can be of the form srcdir:objdir
+    if ':' in subdir:
+        srcdir, subdir = subdir.split(':', 1)
+    else:
+        srcdir = subdir
+    srcdir = os.path.join(topsrcdir, srcdir)
+    objdir = os.path.abspath(subdir)
+
+    return prepare(CONFIGURE_DATA, srcdir, objdir, args[3], args[4:])
+
 
 if __name__ == '__main__':
-    if sys.argv[1] == 'dump':
-        dump(CONFIG_DUMP, sys.argv[2], sys.argv[3:])
-    elif sys.argv[1] == 'adjust':
-        adjust(CONFIG_DUMP, sys.argv[2] if len(sys.argv) > 2 else None)
+    sys.exit(main(sys.argv[1:]))
new file mode 100644
--- /dev/null
+++ b/build/win32/dumpenv4python.pl
@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# See build/autoconf/hooks.m4
+
+use Data::Dumper;
+
+$Data::Dumper::Terse = 1;
+$Data::Dumper::Indent = 0;
+
+# We can't use perl hashes because Mozilla-Build's perl is 5.6, and perl
+# 5.6's Data::Dumper doesn't have Pair to change ' => ' into ' : '.
+@data = (
+  ['env', [map { [$_, $ENV{$_}] } keys %ENV]],
+  ['args', \@ARGV],
+);
+
+print Dumper \@data;