author Brian Hackett <>
Wed, 14 Nov 2018 16:09:58 -1000
changeset 446931 1c7fc8389e012c987347efefca6b35f3948b742a
parent 422876 a8e245778a34a18bd77b0acd62a0172de53a3e30
child 453444 65511e8d7a7e0df5df7c4b7c96ffc3cbd2590db3
permissions -rw-r--r--
Bug 1507359 Part 2 - Bindings and internal changes to allow ReplayDebugger to control child pausing/resuming, r=mccr8.

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at

# This script is used to capture the content of config.status-generated
# files and subsequently restore their timestamp if they haven't changed.

import argparse
import errno
import os
import re
import subprocess
import sys
import pickle

import mozpack.path as mozpath

class File(object):
    def __init__(self, path):
        self._path = path
        self._content = open(path, 'rb').read()
        stat = os.stat(path)
        self._times = (stat.st_atime, stat.st_mtime)

    def path(self):
        return self._path

    def mtime(self):
        return self._times[1]

    def modified(self):
        '''Returns whether the file was modified since the instance was
        created. Result is memoized.'''
        if hasattr(self, '_modified'):
            return self._modified

        modified = True
        if os.path.exists(self._path):
            if open(self._path, 'rb').read() == self._content:
                modified = False
        self._modified = modified
        return modified

    def update_time(self):
        '''If the file hasn't changed since the instance was created,
           restore its old modification time.'''
        if not self.modified:
            os.utime(self._path, self._times)

# As defined in the various sub-configures in the tree

CONFIGURE_DATA = 'configure.pkl'

# Autoconf, in some of the sub-configures used in the tree, likes to error
# out when "precious" variables change in value. The solution it gives to
# straighten things is to either run make distclean or remove config.cache.
# There's no reason not to do the latter automatically instead of failing,
# doing the cleanup (which, on buildbots means a full clobber), and
# restarting from scratch.
def maybe_clear_cache(data):
    env = dict(data['env'])
    for kind in ('target', 'host', 'build'):
        arg = data[kind]
        if arg is not None:
            env['%s_alias' % kind] = arg
    # configure can take variables assignments in its arguments, and that
    # overrides whatever is in the environment.
    for arg in data['args']:
        if arg[:1] != '-' and '=' in arg:
            key, value = arg.split('=', 1)
            env[key] = value

    comment = re.compile(r'^\s+#')
    cache = {}
    with open(data['cache-file']) as f:
        for line in f:
            if not comment.match(line) and '=' in line:
                key, value = line.rstrip(os.linesep).split('=', 1)
                # If the value is quoted, unquote it
                if value[:1] == "'":
                    value = value[1:-1].replace("'\\''", "'")
                cache[key] = value
    for precious in PRECIOUS_VARS:
        # If there is no entry at all for that precious variable, then
        # its value is not precious for that particular configure.
        if 'ac_cv_env_%s_set' % precious not in cache:
        is_set = cache.get('ac_cv_env_%s_set' % precious) == 'set'
        value = cache.get('ac_cv_env_%s_value' % precious) if is_set else None
        if value != env.get(precious):
            print('Removing %s because of %s value change from:' % (data['cache-file'], precious))
            print('  %s' % (value if value is not None else 'undefined'))
            print('  %s' % env.get(precious, 'undefined'))
            return True
    return False

def split_template(s):
    """Given a "file:template" string, returns "file", "template". If the string
    is of the form "file" (without a template), returns "file", ""."""
    if ':' in s:
        return s.split(':', 1)
    return s, '' % s

def get_config_files(data):
    # config.status in js/src never contains the output we try to scan here.
    if data['relobjdir'] == 'js/src':
        return [], []

    config_status = mozpath.join(data['objdir'], 'config.status')
    if not os.path.exists(config_status):
        return [], []

    config_files = []
    command_files = []

    # Scan the config.status output for information about configuration files
    # it generates.
    config_status_output = subprocess.check_output(
        [data['shell'], '-c', '%s --help' % config_status],
    state = None
    for line in config_status_output:
        if line.startswith('Configuration') and line.endswith(':'):
            if line.endswith('commands:'):
                state = 'commands'
                state = 'config'
        elif not line.strip():
            state = None
        elif state:
            for f, t in (split_template(couple) for couple in line.split()):
                f = mozpath.join(data['objdir'], f)
                t = mozpath.join(data['srcdir'], t)
                if state == 'commands':
                    config_files.append((f, t))

    return config_files, command_files

def prepare(srcdir, objdir, shell, args):
    parser = argparse.ArgumentParser()
    parser.add_argument('--target', type=str)
    parser.add_argument('--host', type=str)
    parser.add_argument('--build', type=str)
    parser.add_argument('--cache-file', type=str)
    # The --srcdir argument is simply ignored. It's a useless autoconf feature
    # that we don't support well anyways. This makes it stripped from `others`
    # and allows to skip setting it when calling the subconfigure (configure
    # will take it from the configure path anyways).
    parser.add_argument('--srcdir', type=str)

    data_file = os.path.join(objdir, CONFIGURE_DATA)
    previous_args = None
    if os.path.exists(data_file):
        with open(data_file, 'rb') as f:
            data = pickle.load(f)
            previous_args = data['args']

    # Msys likes to break environment variables and command line arguments,
    # so read those from stdin, as they are passed from the configure script
    # when necessary (on windows).
    input =
    if input:
        data = {a: b for [a, b] in eval(input)}
        environ = {a: b for a, b in data['env']}
        # These environment variables as passed from old-configure may contain
        # posix-style paths, which will not be meaningful to the js
        # subconfigure, which runs as a native python process, so use their
        # values from the environment. In the case of autoconf implemented
        # subconfigures, Msys will re-convert them properly.
        for var in ('HOME', 'TERM', 'PATH', 'TMPDIR', 'TMP',
                    'TEMP', 'INCLUDE'):
            if var in environ and var in os.environ:
                environ[var] = os.environ[var]
        args = data['args']
        environ = os.environ

    args, others = parser.parse_known_args(args)

    data = {
        'args': others,
        'shell': shell,
        'srcdir': srcdir,
        'env': environ,

    if args.cache_file:
        data['cache-file'] = mozpath.normpath(mozpath.join(os.getcwd(),
        data['cache-file'] = mozpath.join(objdir, 'config.cache')

    if previous_args is not None:
        data['previous-args'] = previous_args

    except OSError as e:
        if e.errno != errno.EEXIST:

    with open(data_file, 'wb') as f:
        pickle.dump(data, f)

def prefix_lines(text, prefix):
    return ''.join('%s> %s' % (prefix, line) for line in text.splitlines(True))

def execute_and_prefix(*args, **kwargs):
    prefix = kwargs['prefix']
    del kwargs['prefix']
    proc = subprocess.Popen(*args, stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT, **kwargs)
    while True:
        line = proc.stdout.readline()
        if not line:
        print(prefix_lines(line.rstrip(), prefix))
    return proc.wait()

def run(objdir):
    ret = 0

    with open(os.path.join(objdir, CONFIGURE_DATA), 'rb') as f:
        data = pickle.load(f)

    data['objdir'] = objdir
    relobjdir = data['relobjdir'] = os.path.relpath(objdir, os.getcwd())

    cache_file = data['cache-file']
    cleared_cache = True
    if os.path.exists(cache_file):
        cleared_cache = maybe_clear_cache(data)

    config_files, command_files = get_config_files(data)
    contents = []
    for f, t in config_files:

    # AC_CONFIG_COMMANDS actually only registers tags, not file names
    # but most commands are tagged with the file name they create.
    # However, a few don't, or are tagged with a directory name (and their
    # command is just to create that directory)
    for f in command_files:
        if os.path.isfile(f):

    # Only run configure if one of the following is true:
    # - config.status doesn't exist
    # - config.status is older than an input to configure
    # - the configure arguments changed
    # - the environment changed in a way that requires a cache clear.
    configure = mozpath.join(data['srcdir'], 'old-configure')
    config_status_path = mozpath.join(objdir, 'config.status')
    skip_configure = True
    if not os.path.exists(config_status_path):
        skip_configure = False
        config_status = None
        config_status = File(config_status_path)
        config_status_deps = mozpath.join(objdir, '')
        if not os.path.exists(config_status_deps):
            skip_configure = False
            with open(config_status_deps, 'r') as fh:
                dep_files = + [configure]
            if (any(not os.path.exists(f) or (config_status.mtime < os.path.getmtime(f))
                    for f in dep_files) or
                data.get('previous-args', data['args']) != data['args'] or
                skip_configure = False

    if not skip_configure:
        if mozpath.normsep(relobjdir) == 'js/src':
            # Because configure is a shell script calling a python script
            # calling a shell script, on Windows, with msys screwing the
            # environment, we lose the benefits from our own efforts in this
            # script to get past the msys problems. So manually call the python
            # script instead, so that we don't do a native->msys transition
            # here. Then the python configure will still have the right
            # environment when calling the shell configure.
            command = [
                os.path.join(os.path.dirname(__file__), '..', ''),
            data['env']['OLD_CONFIGURE'] = os.path.join(
                os.path.dirname(configure), 'old-configure')
            command = [data['shell'], configure]
        for kind in ('target', 'build', 'host'):
            if data.get(kind) is not None:
                command += ['--%s=%s' % (kind, data[kind])]
        command += data['args']
        command += ['--cache-file=%s' % cache_file]

        # Pass --no-create to configure so that it doesn't run config.status.
        # We're going to run it ourselves.
        command += ['--no-create']

        print(prefix_lines('configuring', relobjdir))
        print(prefix_lines('running %s' % ' '.join(command[:-1]), relobjdir))
        returncode = execute_and_prefix(command, cwd=objdir, env=data['env'],
        if returncode:
            return returncode

    # Only run config.status if one of the following is true:
    # - config.status changed or did not exist
    # - one of the templates for config files is newer than the corresponding
    #   config file.
    skip_config_status = True
    if mozpath.normsep(relobjdir) == 'js/src':
        # Running config.status in js/src actually does nothing, so we just
        # skip it.
    elif not config_status or config_status.modified:
        # If config.status doesn't exist after configure (because it's not
        # an autoconf configure), skip it.
        if os.path.exists(config_status_path):
            skip_config_status = False
        # config.status changed or was created, so we need to update the
        # list of config and command files.
        config_files, command_files = get_config_files(data)
        for f, t in config_files:
            if not os.path.exists(t) or \
                    os.path.getmtime(f) < os.path.getmtime(t):
                skip_config_status = False

    if not skip_config_status:
        if skip_configure:
            print(prefix_lines('running config.status', relobjdir))
        ret = execute_and_prefix([data['shell'], '-c', './config.status'],
                                 cwd=objdir, env=data['env'], prefix=relobjdir)

        for f in contents:

    return ret

def subconfigure(args):
    parser = argparse.ArgumentParser()
    parser.add_argument('--list', type=str,
                        help='File containing a list of subconfigures to run')
    parser.add_argument('--skip', type=str,
                        help='File containing a list of Subconfigures to skip')
    parser.add_argument('subconfigures', type=str, nargs='*',
                        help='Subconfigures to run if no list file is given')
    args, others = parser.parse_known_args(args)
    subconfigures = args.subconfigures
    if args.list:
        subconfigures.extend(open(args.list, 'rb').read().splitlines())
    if args.skip:
        skips = set(open(args.skip, 'rb').read().splitlines())
        subconfigures = [s for s in subconfigures if s not in skips]

    if not subconfigures:
        return 0

    ret = 0
    for subconfigure in subconfigures:
        returncode = run(subconfigure)
        ret = max(returncode, ret)
        if ret:
    return ret

def main(args):
    if args[0] != '--prepare':
        return subconfigure(args)

    topsrcdir = os.path.abspath(args[1])
    subdir = args[2]
    # subdir can be of the form srcdir:objdir
    if ':' in subdir:
        srcdir, subdir = subdir.split(':', 1)
        srcdir = subdir
    srcdir = os.path.join(topsrcdir, srcdir)
    objdir = os.path.abspath(subdir)

    return prepare(srcdir, objdir, args[3], args[4:])

if __name__ == '__main__':