Bug 1091505 - Run subconfigures in parallel. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 04 Nov 2014 13:50:18 +0900
changeset 240257 6cd60e5f3d5272fc6c01a0c7b9388ae4f107a517
parent 240256 7b1c4a66ad0a642ea26933f1823326d0c4127d3e
child 240258 54d05732f29bb7f9b693fd42bfbd655b94f2115a
child 240320 221ca54f4df091b39f197f6805f4bfbd2d7af893
push id660
push userraliiev@mozilla.com
push dateWed, 18 Feb 2015 20:30:48 +0000
treeherdermozilla-release@49e493494178 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1091505
milestone36.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1091505 - Run subconfigures in parallel. r=gps On automation, this brings Windows configure time on a clobber from 5:30 to 3:10. Sadly, because make needs to run under intl/icu/host before configuring intl/icu/target, intl/icu/host needs to be configured independently. Fortunately, that's not configured for normal windows builds anyways. Also, having multiple subconfigures sharing the same cache file is dangerously racy. Fortunately, not a lot do. In fact, only js/src and $_subconfigure_subdir do, so force the latter (only used for ldap sdk on comm-central) not to configure in parallel.
build/autoconf/config.status.m4
build/autoconf/hooks.m4
build/autoconf/icu.m4
build/subconfigure.py
configure.in
--- a/build/autoconf/config.status.m4
+++ b/build/autoconf/config.status.m4
@@ -192,16 +192,19 @@ fi
 
 changequote([, ])
 
 chmod +x $CONFIG_STATUS
 ])
 
 define([MOZ_RUN_CONFIG_STATUS],
 [
+
+MOZ_RUN_ALL_SUBCONFIGURES()
+
 rm -fr confdefs* $ac_clean_files
 dnl Execute config.status, unless --no-create was passed to configure.
 if test "$no_create" != yes && ! ${PYTHON} $CONFIG_STATUS; then
     trap '' EXIT
     exit 1
 fi
 ])
 
--- a/build/autoconf/hooks.m4
+++ b/build/autoconf/hooks.m4
@@ -11,16 +11,17 @@ changequote([, ])dnl
 ])
 
 dnl Wrap AC_INIT_PREPARE to add the above trap.
 define([_MOZ_AC_INIT_PREPARE], defn([AC_INIT_PREPARE]))
 define([AC_INIT_PREPARE],
 [_MOZ_AC_INIT_PREPARE($1)
 MOZ_CONFIG_LOG_TRAP
 > subconfigures
+> skip_subconfigures
 ])
 
 define([AC_OUTPUT_SUBDIRS],
 [for moz_config_dir in $1; do
   _CONFIG_SHELL=${CONFIG_SHELL-/bin/sh}
   case "$moz_config_dir" in
   *:*)
     objdir=$(echo $moz_config_dir | awk -F: '{print [$]2}')
@@ -48,22 +49,39 @@ define([AC_OUTPUT_SUBDIRS],
     dnl storing the environment and command line arguments from a msys process
     dnl (perl), and reading it from python.
     dumpenv="$PERL $_topsrcdir/build/win32/dumpenv4python.pl $ac_configure_args | "
     ;;
   esac
 
   eval $dumpenv $PYTHON $_topsrcdir/build/subconfigure.py --prepare "$srcdir" "$moz_config_dir" "$_CONFIG_SHELL" $ac_configure_args ifelse($2,,,--cache-file="$2")
 
-  dnl Execute subconfigure, unless --no-recursion was passed to configure.
-  if test "$no_recursion" != yes; then
-    trap '' EXIT
-    if ! $PYTHON $_topsrcdir/build/subconfigure.py "$objdir"; then
-        exit 1
-    fi
-    MOZ_CONFIG_LOG_TRAP
-  fi
+  dnl Actual subconfigure execution happens in MOZ_RUN_CONFIG_STATUS
+done
+])
+
+define([AC_OUTPUT_SUBDIRS_NOW],
+[
+for moz_config_dir_ in $1; do
+  AC_OUTPUT_SUBDIRS($moz_config_dir_,$2)
+  tail -1 subconfigures >> skip_subconfigures
+  MOZ_RUN_SUBCONFIGURES(`tail -1 skip_subconfigures`)
 done
 ])
 
+define([MOZ_RUN_SUBCONFIGURES],
+[dnl Execute subconfigure, unless --no-recursion was passed to configure.
+if test "$no_recursion" != yes; then
+  trap '' EXIT
+  if ! $PYTHON $_topsrcdir/build/subconfigure.py $1; then
+      exit 1
+  fi
+  MOZ_CONFIG_LOG_TRAP
+fi
+])
+
+define([MOZ_RUN_ALL_SUBCONFIGURES],
+MOZ_RUN_SUBCONFIGURES([--list subconfigures --skip skip_subconfigures])
+)
+
 dnl Print error messages in config.log as well as stderr
 define([AC_MSG_ERROR],
 [{ echo "configure: error: $1" 1>&2; echo "configure: error: $1" 1>&5; exit 1; }])
--- a/build/autoconf/icu.m4
+++ b/build/autoconf/icu.m4
@@ -175,17 +175,17 @@ if test -z "$BUILDING_JS" -o -n "$JS_STA
          export CPP="$HOST_CPP"
          export LD="$HOST_LD"
          export CFLAGS="$HOST_ICU_CFLAGS $HOST_OPTIMIZE_FLAGS"
          export CPPFLAGS="$ICU_CPPFLAGS"
          export CXXFLAGS="$HOST_ICU_CXXFLAGS $HOST_OPTIMIZE_FLAGS"
          export LDFLAGS="$HOST_LDFLAGS"
          ac_configure_args="$HOST_ICU_BUILD_OPTS"
          ac_configure_args="$ac_configure_args --enable-static --disable-shared --enable-extras=no --enable-icuio=no --enable-layout=no --enable-tests=no --enable-samples=no"
-         AC_OUTPUT_SUBDIRS(intl/icu/source:intl/icu/host)
+         AC_OUTPUT_SUBDIRS_NOW(intl/icu/source:intl/icu/host)
         ) || exit 1
     	# generate config/icucross.mk
     	$GMAKE -C $_objdir/intl/icu/host/ config/icucross.mk
 
     	# --with-cross-build requires absolute path
     	ICU_HOST_PATH=`cd $_objdir/intl/icu/host && pwd`
     	ICU_CROSS_BUILD_OPT="--with-cross-build=$ICU_HOST_PATH --disable-tools"
     	ICU_TARGET_OPT="--build=$build --host=$target"
--- a/build/subconfigure.py
+++ b/build/subconfigure.py
@@ -10,16 +10,38 @@ import errno
 import os
 import re
 import subprocess
 import sys
 import pickle
 
 import mozpack.path as mozpath
 
+try:
+    from multiprocessing import Pool, cpu_count
+except ImportError:
+    import itertools
+
+    class Pool(object):
+        def __init__(self, size):
+            pass
+
+        def imap_unordered(self, fn, iterable):
+            return itertools.imap(fn, iterable)
+
+        def close(self):
+            pass
+
+        def join(self):
+            pass
+
+    def cpu_count():
+        return 1
+
+
 class File(object):
     def __init__(self, path):
         self._path = path
         self._content = open(path, 'rb').read()
         stat = os.stat(path)
         self._times = (stat.st_atime, stat.st_mtime)
 
     @property
@@ -65,16 +87,19 @@ PRECIOUS_VARS = set([
     'CCC',
     'CXXFLAGS',
     'CXX',
     'CCASFLAGS',
     'CCAS',
 ])
 
 
+CONFIGURE_DATA = 'configure.pkl'
+
+
 # Autoconf, in some of the sub-configures used in the tree, likes to error
 # out when "precious" variables change in value. The solution it gives to
 # straighten things is to either run make distclean or remove config.cache.
 # There's no reason not to do the latter automatically instead of failing,
 # doing the cleanup (which, on buildbots means a full clobber), and
 # restarting from scratch.
 def maybe_clear_cache(data):
     env = dict(data['env'])
@@ -155,29 +180,29 @@ def get_config_files(data):
                 if state == 'commands':
                     command_files.append(f)
                 else:
                     config_files.append((f, t))
 
     return config_files, command_files
 
 
-def prepare(data_file, srcdir, objdir, shell, args):
+def prepare(srcdir, objdir, shell, args):
     parser = argparse.ArgumentParser()
     parser.add_argument('--target', type=str)
     parser.add_argument('--host', type=str)
     parser.add_argument('--build', type=str)
     parser.add_argument('--cache-file', type=str)
     # The --srcdir argument is simply ignored. It's a useless autoconf feature
     # that we don't support well anyways. This makes it stripped from `others`
     # and allows to skip setting it when calling the subconfigure (configure
     # will take it from the configure path anyways).
     parser.add_argument('--srcdir', type=str)
 
-    data_file = os.path.join(objdir, data_file)
+    data_file = os.path.join(objdir, CONFIGURE_DATA)
     previous_args = None
     if os.path.exists(data_file):
         with open(data_file, 'rb') as f:
             data = pickle.load(f)
             previous_args = data['args']
 
     # Msys likes to break environment variables and command line arguments,
     # so read those from stdin, as they are passed from the configure script
@@ -222,20 +247,25 @@ def prepare(data_file, srcdir, objdir, s
     except OSError as e:
         if e.errno != errno.EEXIST:
             raise
 
     with open(data_file, 'wb') as f:
         pickle.dump(data, f)
 
 
-def run(data_file, objdir):
+def prefix_lines(text, prefix):
+    return ''.join('%s> %s' % (prefix, line) for line in text.splitlines(True))
+
+
+def run(objdir):
     ret = 0
+    output = ''
 
-    with open(os.path.join(objdir, data_file), 'rb') as f:
+    with open(os.path.join(objdir, CONFIGURE_DATA), 'rb') as f:
         data = pickle.load(f)
 
     data['objdir'] = objdir
 
     cache_file = data['cache-file']
     cleared_cache = True
     if os.path.exists(cache_file):
         cleared_cache = maybe_clear_cache(data)
@@ -266,35 +296,38 @@ def run(data_file, objdir):
         config_status = None
     else:
         config_status = File(config_status_path)
         if config_status.mtime < os.path.getmtime(configure) or \
                 data.get('previous-args', data['args']) != data['args'] or \
                 cleared_cache:
             skip_configure = False
 
+    relobjdir = os.path.relpath(objdir, os.getcwd())
+
     if not skip_configure:
         command = [data['shell'], configure]
         for kind in ('target', 'build', 'host'):
             if data.get(kind) is not None:
                 command += ['--%s=%s' % (kind, data[kind])]
         command += data['args']
         command += ['--cache-file=%s' % cache_file]
 
         # Pass --no-create to configure so that it doesn't run config.status.
         # We're going to run it ourselves.
         command += ['--no-create']
 
-        print 'configuring in %s' % os.path.relpath(objdir, os.getcwd())
-        print 'running %s' % ' '.join(command[:-1])
+        print prefix_lines('configuring', relobjdir)
+        print prefix_lines('running %s' % ' '.join(command[:-1]), relobjdir)
         sys.stdout.flush()
-        ret = subprocess.call(command, cwd=objdir, env=data['env'])
-
-        if ret:
-            return ret
+        try:
+            output += subprocess.check_output(command,
+                stderr=subprocess.STDOUT, cwd=objdir, env=data['env'])
+        except subprocess.CalledProcessError as e:
+            return relobjdir, e.returncode, e.output
 
         # Leave config.status with a new timestamp if configure is newer than
         # its original mtime.
         if config_status and os.path.getmtime(configure) <= config_status.mtime:
             config_status.update_time()
 
     # Only run config.status if one of the following is true:
     # - config.status changed or did not exist
@@ -312,43 +345,79 @@ def run(data_file, objdir):
         config_files, command_files = get_config_files(data)
         for f, t in config_files:
             if not os.path.exists(t) or \
                     os.path.getmtime(f) < os.path.getmtime(t):
                 skip_config_status = False
 
     if not skip_config_status:
         if skip_configure:
-            print 'running config.status in %s' % os.path.relpath(objdir,
-                os.getcwd())
+            print prefix_lines('running config.status', relobjdir)
             sys.stdout.flush()
-        ret = subprocess.call([data['shell'], '-c', './config.status'],
-            cwd=objdir, env=data['env'])
+        try:
+            output += subprocess.check_output([data['shell'], '-c',
+                './config.status'], stderr=subprocess.STDOUT, cwd=objdir,
+                env=data['env'])
+        except subprocess.CalledProcessError as e:
+            ret = e.returncode
+            output += e.output
 
         for f in contents:
             f.update_time()
 
+    return relobjdir, ret, output
+
+
+def subconfigure(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--list', type=str,
+        help='File containing a list of subconfigures to run')
+    parser.add_argument('--skip', type=str,
+        help='File containing a list of Subconfigures to skip')
+    parser.add_argument('subconfigures', type=str, nargs='*',
+        help='Subconfigures to run if no list file is given')
+    args, others = parser.parse_known_args(args)
+    subconfigures = args.subconfigures
+    if args.list:
+        subconfigures.extend(open(args.list, 'rb').read().splitlines())
+    if args.skip:
+        skips = set(open(args.skip, 'rb').read().splitlines())
+        subconfigures = [s for s in subconfigures if s not in skips]
+
+    if not subconfigures:
+        return 0
+
+    ret = 0
+    # One would think using a ThreadPool would be faster, considering
+    # everything happens in subprocesses anyways, but no, it's actually
+    # slower on Windows. (20s difference overall!)
+    pool = Pool(min(len(subconfigures), cpu_count()))
+    for relobjdir, returncode, output in \
+            pool.imap_unordered(run, subconfigures):
+        print prefix_lines(output, relobjdir)
+        sys.stdout.flush()
+        ret = max(returncode, ret)
+        if ret:
+            break
+    pool.close()
+    pool.join()
     return ret
 
 
-CONFIGURE_DATA = 'configure.pkl'
-
 def main(args):
     if args[0] != '--prepare':
-        if len(args) != 1:
-            raise Exception('Usage: %s relativeobjdir' % __file__)
-        return run(CONFIGURE_DATA, args[0])
+        return subconfigure(args)
 
     topsrcdir = os.path.abspath(args[1])
     subdir = args[2]
     # subdir can be of the form srcdir:objdir
     if ':' in subdir:
         srcdir, subdir = subdir.split(':', 1)
     else:
         srcdir = subdir
     srcdir = os.path.join(topsrcdir, srcdir)
     objdir = os.path.abspath(subdir)
 
-    return prepare(CONFIGURE_DATA, srcdir, objdir, args[3], args[4:])
+    return prepare(srcdir, objdir, args[3], args[4:])
 
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv[1:]))
--- a/configure.in
+++ b/configure.in
@@ -9235,17 +9235,17 @@ unset MAKEFILES
 unset CONFIG_FILES
 
 # Run all configure scripts specified by a subconfigure
 if test -n "$_subconfigure_subdir"; then
   _save_srcdir="$srcdir"
   srcdir="$srcdir/.."
   _save_ac_configure_args="$ac_configure_args"
   ac_configure_args="$_subconfigure_config_args"
-  AC_OUTPUT_SUBDIRS("$_subconfigure_subdir",$cache_file)
+  AC_OUTPUT_SUBDIRS_NOW("$_subconfigure_subdir",$cache_file)
   ac_configure_args="$_save_ac_configure_args"
   srcdir="$_save_srcdir"
 fi
 
 # No need to run subconfigures when building with LIBXUL_SDK_DIR
 if test "$COMPILE_ENVIRONMENT" -a -z "$LIBXUL_SDK_DIR"; then
 
 export WRAP_LDFLAGS