Bug 1256571 - Change the execution model of python configure. r=chmanchester
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 12 Apr 2016 21:32:38 +0900
changeset 330989 25f299dca206e35c03b99a3fdac12b6482dc90c0
parent 330988 02561278899ca2262e62fe7060cb24e0a80f9293
child 330990 3e17cc4625da9c00f7815f215d89bd3b1af68e2d
push id6048
push userkmoir@mozilla.com
push dateMon, 06 Jun 2016 19:02:08 +0000
treeherdermozilla-beta@46d72a56c57d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1256571
milestone48.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1256571 - Change the execution model of python configure. r=chmanchester So far, everything was essentially executed at "declaration". This made the sandbox code simpler, but to improve on the tooling around python configure (for tests and introspection), we need to have more flexibility, which executing everything at declaration doesn't give. With this change, only @depends functions depending on --help, as well as templates, are executed at the moment the moz.configure files are included in the sandbox. The remainder is executed at the end.
build/moz.configure/init.configure
python/mozbuild/mozbuild/configure/__init__.py
python/mozbuild/mozbuild/test/configure/test_checks_configure.py
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -156,25 +156,28 @@ def add_old_configure_arg(arg):
         if arg:
             args.append(arg)
 
 
 option(env='PYTHON', nargs=1, help='Python interpreter')
 
 # Setup python virtualenv
 # ==============================================================
-@depends('PYTHON', check_build_environment, mozconfig)
+@depends('PYTHON', check_build_environment, mozconfig, '--help')
 @imports('os')
 @imports('sys')
 @imports('subprocess')
 @imports(_from='mozbuild.configure.util', _import='LineIO')
 @imports(_from='mozbuild.virtualenv', _import='VirtualenvManager')
 @imports(_from='mozbuild.virtualenv', _import='verify_python_version')
 @imports('distutils.sysconfig')
-def virtualenv_python(env_python, build_env, mozconfig):
+def virtualenv_python(env_python, build_env, mozconfig, help):
+    if help:
+        return
+
     python = env_python[0] if env_python else None
 
     # Ideally we'd rely on the mozconfig injection from mozconfig_options,
     # but we'd rather avoid the verbosity when we need to reexecute with
     # a different python.
     if mozconfig['path']:
         if 'PYTHON' in mozconfig['env']['added']:
             python = mozconfig['env']['added']['PYTHON']
--- a/python/mozbuild/mozbuild/configure/__init__.py
+++ b/python/mozbuild/mozbuild/configure/__init__.py
@@ -124,16 +124,19 @@ class ConfigureSandbox(dict):
         # Store options added with `imply_option`, and the reason they were
         # added (which can either have been given to `imply_option`, or
         # inferred. Their order matters, so use a list.
         self._implied_options = []
 
         # Store all results from _prepare_function
         self._prepared_functions = set()
 
+        # Queue of functions to execute, with their arguments
+        self._execution_queue = []
+
         self._helper = CommandLineHelper(environ, argv)
 
         assert isinstance(config, dict)
         self._config = config
 
         if logger is None:
             logger = moz_logger = logging.getLogger('moz.configure')
             logger.setLevel(logging.DEBUG)
@@ -166,17 +169,20 @@ class ConfigureSandbox(dict):
             self._help.add(self._help_option)
         elif moz_logger:
             handler = logging.FileHandler('config.log', mode='w', delay=True)
             handler.setFormatter(formatter)
             logger.addHandler(handler)
 
     def include_file(self, path):
         '''Include one file in the sandbox. Users of this class probably want
-        to use `run` instead.'''
+
+        Note: this will execute all template invocations, as well as @depends
+        functions that depend on '--help', but nothing else.
+        '''
 
         if self._paths:
             path = mozpath.join(mozpath.dirname(self._paths[-1]), path)
             path = mozpath.normpath(path)
             if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)):
                 raise ConfigureError(
                     'Cannot include `%s` because it is not in a subdirectory '
                     'of `%s`' % (path, mozpath.dirname(self._paths[0])))
@@ -192,48 +198,48 @@ class ConfigureSandbox(dict):
 
         code = compile(source, path, 'exec')
 
         exec(code, self)
 
         self._paths.pop(-1)
 
     def run(self, path=None):
-        '''Executes the given file within the sandbox, and ensure the overall
-        consistency of the executed script.'''
+        '''Executes the given file within the sandbox, as well as everything
+        pending from any other included file, and ensure the overall
+        consistency of the executed script(s).'''
         if path:
             self.include_file(path)
 
         for option in self._options.itervalues():
             # All options must be referenced by some @depends function
             if option not in self._seen:
                 raise ConfigureError(
                     'Option `%s` is not handled ; reference it with a @depends'
                     % option.option
                 )
 
-            # When running with --help, few options are handled but we still
-            # want to find the unknown ones below, so handle them all now. We
-            # however don't run any of the @depends function that depend on
-            # them.
-            if self._help:
-                self._helper.handle(option)
+            self._value_for(option)
 
         # All implied options should exist.
         for implied_option in self._implied_options:
             raise ConfigureError(
                 '`%s`, emitted from `%s` line %d, is unknown.'
                 % (implied_option.option, implied_option.caller[1],
                    implied_option.caller[2]))
 
         # All options should have been removed (handled) by now.
         for arg in self._helper:
             without_value = arg.split('=', 1)[0]
             raise InvalidOptionError('Unknown option: %s' % without_value)
 
+        # Run the execution queue
+        for func, args in self._execution_queue:
+            func(*args)
+
         if self._help:
             with LineIO(self.log_impl.info) as out:
                 self._help.usage(out)
 
     def __getitem__(self, key):
         impl = '%s_impl' % key
         func = getattr(self, impl, None)
         if func:
@@ -286,16 +292,19 @@ class ConfigureSandbox(dict):
             for arg in dependencies:
                 if isinstance(arg, DependsFunction):
                     _, deps = self._depends[arg]
                     if self._help_option not in deps:
                         raise ConfigureError(
                             "`%s` depends on '--help' and `%s`. "
                             "`%s` must depend on '--help'"
                             % (func.__name__, arg.__name__, arg.__name__))
+        elif self._help:
+            raise ConfigureError("Missing @depends for `%s`: '--help'" %
+                                 func.__name__)
 
         resolved_args = [self._value_for(d) for d in dependencies]
         return func(*resolved_args)
 
     @memoize
     def _value_for_option(self, option):
         implied = {}
         for implied_option in self._implied_options[:]:
@@ -358,18 +367,16 @@ class ConfigureSandbox(dict):
         if option.name:
             self._options[option.name] = option
         if option.env:
             self._options[option.env] = option
 
         if self._help:
             self._help.add(option)
 
-        self._value_for(option)
-
         return option
 
     def depends_impl(self, *args):
         '''Implementation of @depends()
         This function is a decorator. It returns a function that subsequently
         takes a function and returns a dummy function. The dummy function
         identifies the actual function for the sandbox, while preventing
         further function calls from within the sandbox.
@@ -412,18 +419,24 @@ class ConfigureSandbox(dict):
 
         def decorator(func):
             if inspect.isgeneratorfunction(func):
                 raise ConfigureError(
                     'Cannot decorate generator functions with @depends')
             func, glob = self._prepare_function(func)
             dummy = wraps(func)(DependsFunction())
             self._depends[dummy] = func, dependencies
-            if not self._help or self._help_option in dependencies:
+
+            # Only @depends functions with a dependency on '--help' are
+            # executed immediately. Everything else is queued for later
+            # execution.
+            if self._help_option in dependencies:
                 self._value_for(dummy)
+            elif not self._help:
+                self._execution_queue.append((self._value_for, (dummy,)))
 
             return dummy
 
         return decorator
 
     def include_impl(self, what):
         '''Implementation of include().
         Allows to include external files for execution in the sandbox.
@@ -567,28 +580,30 @@ class ConfigureSandbox(dict):
 
     def set_config_impl(self, name, value):
         '''Implementation of set_config().
         Set the configuration items with the given name to the given value.
         Both `name` and `value` can be references to @depends functions,
         in which case the result from these functions is used. If the result
         of either function is None, the configuration item is not set.
         '''
-        self._resolve_and_set(self._config, name, value)
+        self._execution_queue.append((
+            self._resolve_and_set, (self._config, name, value)))
 
     def set_define_impl(self, name, value):
         '''Implementation of set_define().
         Set the define with the given name to the given value. Both `name` and
         `value` can be references to @depends functions, in which case the
         result from these functions is used. If the result of either function
         is None, the define is not set. If the result is False, the define is
         explicitly undefined (-U).
         '''
         defines = self._config.setdefault('DEFINES', {})
-        self._resolve_and_set(defines, name, value)
+        self._execution_queue.append((
+            self._resolve_and_set, (defines, name, value)))
 
     def imply_option_impl(self, option, value, reason=None):
         '''Implementation of imply_option().
         Injects additional options as if they had been passed on the command
         line. The `option` argument is a string as in option()'s `name` or
         `env`. The option must be declared after `imply_option` references it.
         The `value` argument indicates the value to pass to the option.
         It can be:
--- a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py
@@ -159,16 +159,17 @@ class TestChecksConfigure(unittest.TestC
         sandbox = FindProgramSandbox(config, environ, [prog] + args, out, out)
         base_dir = os.path.join(topsrcdir, 'build', 'moz.configure')
         sandbox.include_file(os.path.join(base_dir, 'util.configure'))
         sandbox.include_file(os.path.join(base_dir, 'checks.configure'))
 
         status = 0
         try:
             exec(command, sandbox)
+            sandbox.run()
         except SystemExit as e:
             status = e.code
 
         return config, out.getvalue(), status
 
     def test_check_prog(self):
         config, out, status = self.get_result(
             'check_prog("FOO", ("known-a",))')