Bug 969164 - Use per-directory config in sandboxes when reading moz.builds. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 11 Feb 2014 10:37:47 +0900
changeset 168008 06f5f22f1bb9d738849fc16bf99108ed641a2e9e
parent 168007 4a90387184174c673332acd5e878ab1dda5dbf24
child 168009 2f0f52f627d71c2676d8f7c34830e0d03d6fa3e6
push id26194
push userryanvm@gmail.com
push dateTue, 11 Feb 2014 13:39:41 +0000
treeherdermozilla-central@edd724161ab4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 969164 - Use per-directory config in sandboxes when reading moz.builds. r=gps
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -164,34 +164,16 @@ class BuildBackend(LoggingMixin):
     def _init():
         """Hook point for child classes to perform actions during __init__.
         This exists so child classes don't need to implement __init__.
-    def get_environment(self, obj):
-        """Obtain the ConfigEnvironment for a specific object.
-        This is used to support external source directories which operate in
-        their own topobjdir and have their own ConfigEnvironment.
-        This is somewhat hacky and should be considered for rewrite if external
-        project integration is rewritten.
-        """
-        environment = self._environments.get(obj.topobjdir, None)
-        if not environment:
-            config_status = mozpath.join(obj.topobjdir, 'config.status')
-            environment = ConfigEnvironment.from_config_status(config_status)
-            self._environments[obj.topobjdir] = environment
-        return environment
     def consume(self, objs):
         """Consume a stream of TreeMetadata instances.
         This is the main method of the interface. This is what takes the
         frontend output and does something with it.
         Child classes are not expected to implement this method. Instead, the
         base class consumes objects and calls methods (possibly) implemented by
@@ -305,17 +287,17 @@ class BuildBackend(LoggingMixin):
     def _get_preprocessor(self, obj):
         '''Returns a preprocessor with a few predefined values depending on
         the given BaseConfigSubstitution(-like) object, and all the substs
         in the current environment.'''
         pp = Preprocessor()
         srcdir = mozpath.dirname(obj.input_path)
-        pp.context.update(self.environment.substs)
+        pp.context.update(obj.config.substs)
             relativesrcdir=mozpath.relpath(srcdir, obj.topsrcdir) or '.',
             DEPTH=mozpath.relpath(obj.topobjdir, mozpath.dirname(obj.output_path)) or '.',
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -165,18 +165,21 @@ class TestManager(object):
 class CommonBackend(BuildBackend):
     """Holds logic common to all build backends."""
     def _init(self):
         self._idl_manager = XPIDLManager(self.environment)
         self._test_manager = TestManager(self.environment)
         self._webidls = WebIDLCollection()
+        self._configs = set()
     def consume_object(self, obj):
+        self._configs.add(obj.config)
         if isinstance(obj, TestManifest):
             for test in obj.tests:
                 self._test_manager.add(test, flavor=obj.flavor,
         elif isinstance(obj, XPIDLFile):
             self._idl_manager.register_idl(obj.source_path, obj.module)
@@ -226,16 +229,19 @@ class CommonBackend(BuildBackend):
     def consume_finished(self):
         if len(self._idl_manager.idls):
+        for config in self._configs:
+            self.backend_input_files.add(config.source)
         # Write out a machine-readable file describing every test.
         path = mozpath.join(self.environment.topobjdir, 'all-tests.json')
         with self._write_file(path) as fh:
             json.dump(self._test_manager.tests_by_path, fh, sort_keys=True,
     def _create_config_header(self, obj):
         '''Creates the given config header. A config header is generated by
@@ -252,24 +258,24 @@ class CommonBackend(BuildBackend):
             r = re.compile('^\s*#\s*(?P<cmd>[a-z]+)(?:\s+(?P<name>\S+)(?:\s+(?P<value>\S+))?)?', re.U)
             for l in input:
                 m = r.match(l)
                 if m:
                     cmd = m.group('cmd')
                     name = m.group('name')
                     value = m.group('value')
                     if name:
-                        if name in self.environment.defines:
+                        if name in obj.config.defines:
                             if cmd == 'define' and value:
                                 l = l[:m.start('value')] \
-                                    + str(self.environment.defines[name]) \
+                                    + str(obj.config.defines[name]) \
                                     + l[m.end('value'):]
                             elif cmd == 'undef':
                                 l = l[:m.start('cmd')] \
                                     + 'define' \
                                     + l[m.end('cmd'):m.end('name')] \
                                     + ' ' \
-                                    + str(self.environment.defines[name]) \
+                                    + str(obj.config.defines[name]) \
                                     + l[m.end('name'):]
                         elif cmd == 'undef':
                            l = '/* ' + l[:m.end('name')] + ' */' + l[m.end('name'):]
--- a/python/mozbuild/mozbuild/backend/configenvironment.py
+++ b/python/mozbuild/mozbuild/backend/configenvironment.py
@@ -94,18 +94,21 @@ class ConfigEnvironment(object):
     ConfigEnvironment expects a "top_srcdir" subst to be set with the top
     source directory, in msys format on windows. It is used to derive a
     "srcdir" subst when treating config files. It can either be an absolute
     path or a path relative to the topobjdir.
     def __init__(self, topsrcdir, topobjdir, defines=[], non_global_defines=[],
-            substs=[]):
+        substs=[], source=None):
+        if not source:
+            source = mozpath.join(topobjdir, 'config.status')
+        self.source = source
         self.defines = ReadOnlyDict(defines)
         self.substs = dict(substs)
         self.topsrcdir = mozpath.normsep(topsrcdir)
         self.topobjdir = mozpath.normsep(topobjdir)
         global_defines = [name for name, value in defines
             if not name in non_global_defines]
         self.substs['ACDEFINES'] = ' '.join(['-D%s=%s' % (name,
             shell_quote(self.defines[name]).replace('$', '$$')) for name in global_defines])
@@ -149,9 +152,9 @@ class ConfigEnvironment(object):
         self.substs_unicode = ReadOnlyDict(self.substs_unicode)
     def from_config_status(path):
         config = BuildConfig.from_config_status(path)
         return ConfigEnvironment(config.topsrcdir, config.topobjdir,
-            config.defines, config.non_global_defines, config.substs)
+            config.defines, config.non_global_defines, config.substs, path)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -325,17 +325,17 @@ class RecursiveMakeBackend(CommonBackend
     def consume_object(self, obj):
         """Write out build files necessary to build with recursive make."""
         if not isinstance(obj, SandboxDerived):
         if obj.objdir not in self._backend_files:
             self._backend_files[obj.objdir] = \
-                BackendMakeFile(obj.srcdir, obj.objdir, self.get_environment(obj))
+                BackendMakeFile(obj.srcdir, obj.objdir, obj.config)
         backend_file = self._backend_files[obj.objdir]
         CommonBackend.consume_object(self, obj)
         # CommonBackend handles XPIDLFile and TestManifest, but we want to do
         # some extra things for them.
         if isinstance(obj, XPIDLFile):
@@ -700,16 +700,17 @@ class RecursiveMakeBackend(CommonBackend
                     self.log(logging.DEBUG, 'stub_makefile',
                         {'path': makefile}, 'Creating stub Makefile: {path}')
                 obj = self.Substitution()
                 obj.output_path = makefile
                 obj.input_path = makefile_in
                 obj.topsrcdir = bf.environment.topsrcdir
                 obj.topobjdir = bf.environment.topobjdir
+                obj.config = bf.environment
                 self._create_makefile(obj, stub=stub)
         # Write out a master list of all IPDL source files.
         ipdl_dir = mozpath.join(self.environment.topobjdir, 'ipc', 'ipdl')
         mk = mozmakeutil.Makefile()
         sorted_ipdl_sources = list(sorted(self._ipdl_sources))
         mk.add_statement('ALL_IPDLSRCS := %s' % ' '.join(sorted_ipdl_sources))
@@ -961,16 +962,17 @@ class RecursiveMakeBackend(CommonBackend
         obj = self.Substitution()
         obj.output_path = mozpath.join(self.environment.topobjdir, 'config',
             'makefiles', 'xpidl', 'Makefile')
         obj.input_path = mozpath.join(self.environment.topsrcdir, 'config',
             'makefiles', 'xpidl', 'Makefile.in')
         obj.topsrcdir = self.environment.topsrcdir
         obj.topobjdir = self.environment.topobjdir
+        obj.config = self.environment
         self._create_makefile(obj, extra=dict(
             xpidl_modules=' '.join(xpt_modules),
     def _process_program(self, program, backend_file):
         backend_file.write('PROGRAM = %s\n' % program)
@@ -1086,16 +1088,17 @@ class RecursiveMakeBackend(CommonBackend
     class Substitution(object):
         """BaseConfigSubstitution-like class for use with _create_makefile."""
         __slots__ = (
+            'config',
     def _create_makefile(self, obj, stub=False, extra=None):
         '''Creates the given makefile. Makefiles are treated the same as
         config files, but some additional header and footer is added to the
         When the stub argument is True, no source file is used, and a stub
--- a/python/mozbuild/mozbuild/config_status.py
+++ b/python/mozbuild/mozbuild/config_status.py
@@ -21,17 +21,17 @@ from mozbuild.frontend.emitter import Tr
 from mozbuild.frontend.reader import BuildReader
 from mozbuild.mozinfo import write_mozinfo
 log_manager = LoggingManager()
 def config_status(topobjdir='.', topsrcdir='.',
-        defines=[], non_global_defines=[], substs=[]):
+        defines=[], non_global_defines=[], substs=[], source=None):
     '''Main function, providing config.status functionality.
     Contrary to config.status, it doesn't use CONFIG_FILES or CONFIG_HEADERS
     Without the -n option, this program acts as config.status and considers
     the current directory as the top object directory, even when config.status
     is in a different directory. It will, however, treat the directory
@@ -68,17 +68,17 @@ def config_status(topobjdir='.', topsrcd
                       help='print diffs of changed files.')
     options, args = parser.parse_args()
     # Without -n, the current directory is meant to be the top object directory
     if not options.not_topobjdir:
         topobjdir = os.path.abspath('.')
     env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines,
-            non_global_defines=non_global_defines, substs=substs)
+            non_global_defines=non_global_defines, substs=substs, source=source)
     # mozinfo.json only needs written if configure changes and configure always
     # passes this environment variable.
     if 'WRITE_MOZINFO' in os.environ:
         write_mozinfo(os.path.join(topobjdir, 'mozinfo.json'), env, os.environ)
     reader = BuildReader(env)
     emitter = TreeMetadataEmitter(env)
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -76,16 +76,18 @@ class SandboxDerived(TreeMetadata):
         # Basic directory state.
         self.topsrcdir = sandbox['TOPSRCDIR']
         self.topobjdir = sandbox['TOPOBJDIR']
         self.relativedir = sandbox['RELATIVEDIR']
         self.srcdir = sandbox['SRCDIR']
         self.objdir = sandbox['OBJDIR']
+        self.config = sandbox.config
 class DirectoryTraversal(SandboxDerived):
     """Describes how directory traversal for building should work.
     This build object is likely only of interest to the recursive make backend.
     Other build backends should (ideally) not attempt to mimic the behavior of
     the recursive make backend. The only reason this exists is to support the
     existing recursive make backend while the transition to mozbuild frontend
--- a/python/mozbuild/mozbuild/frontend/gyp_reader.py
+++ b/python/mozbuild/mozbuild/frontend/gyp_reader.py
@@ -115,16 +115,17 @@ def read_from_gyp(config, path, output, 
     # gives us paths normalized with forward slash separator.
     for target in gyp.common.AllTargets(flat_list, targets, path.replace(b'/', os.sep)):
         build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(target)
         # The list of included files returned by gyp are relative to build_file
         included_files = [mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f))
                           for f in data[build_file]['included_files']]
         # Emit a sandbox for each target.
         sandbox = GypSandbox(mozpath.abspath(build_file), included_files)
+        sandbox.config = config
         with sandbox.allow_all_writes() as d:
             topsrcdir = d['TOPSRCDIR'] = config.topsrcdir
             d['TOPOBJDIR'] = config.topobjdir
             relsrcdir = d['RELATIVEDIR'] = mozpath.relpath(mozpath.dirname(build_file), config.topsrcdir)
             d['SRCDIR'] = mozpath.join(topsrcdir, relsrcdir)
             # Each target is given its own objdir. The base of that objdir
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -615,17 +615,17 @@ class BuildReader(object):
         This starts with the tree's top-most moz.build file and descends into
         all linked moz.build files until all relevant files have been evaluated.
         This is a generator of Sandbox instances. As each moz.build file is
         read, a new Sandbox is created and emitted.
         path = mozpath.join(self.topsrcdir, 'moz.build')
-        return self.read_mozbuild(path, read_tiers=True,
+        return self.read_mozbuild(path, self.config, read_tiers=True,
             filesystem_absolute=True, metadata={'tier': None})
     def walk_topsrcdir(self):
         """Read all moz.build files in the source tree.
         This is different from read_topsrcdir() in that this version performs a
         filesystem walk to discover every moz.build file rather than relying on
         data from executed moz.build files to drive traversal.
@@ -643,22 +643,22 @@ class BuildReader(object):
         finder = FileFinder(self.topsrcdir, find_executables=False,
         for path, f in finder.find('**/moz.build'):
             path = os.path.join(self.topsrcdir, path)
-            for s in self.read_mozbuild(path, descend=False,
+            for s in self.read_mozbuild(path, self.config, descend=False,
                 filesystem_absolute=True, read_tiers=True):
                 yield s
-    def read_mozbuild(self, path, read_tiers=False, filesystem_absolute=False,
-            descend=True, metadata={}):
+    def read_mozbuild(self, path, config, read_tiers=False,
+            filesystem_absolute=False, descend=True, metadata={}):
         """Read and process a mozbuild file, descending into children.
         This starts with a single mozbuild file, executes it, and descends into
         other referenced files per our traversal logic.
         The traversal logic is to iterate over the *DIRS variables, treating
         each element as a relative directory path. For each encountered
         directory, we will open the moz.build file located in that
@@ -676,19 +676,19 @@ class BuildReader(object):
         feature is intended to facilitate the build reader injecting state and
         annotations into moz.build files that is independent of the sandbox's
         execution context.
         Traversal is performed depth first (for no particular reason).
-            for s in self._read_mozbuild(path, read_tiers=read_tiers,
-                filesystem_absolute=filesystem_absolute, descend=descend,
-                metadata=metadata):
+            for s in self._read_mozbuild(path, config, read_tiers=read_tiers,
+                filesystem_absolute=filesystem_absolute,
+                descend=descend, metadata=metadata):
                 yield s
         except BuildReaderError as bre:
             raise bre
         except SandboxCalledError as sce:
             raise BuildReaderError(list(self._execution_stack),
                 sys.exc_info()[2], sandbox_called_error=sce)
@@ -704,31 +704,31 @@ class BuildReader(object):
         except SandboxValidationError as ve:
             raise BuildReaderError(list(self._execution_stack),
                 sys.exc_info()[2], validation_error=ve)
         except Exception as e:
             raise BuildReaderError(list(self._execution_stack),
                 sys.exc_info()[2], other_error=e)
-    def _read_mozbuild(self, path, read_tiers, filesystem_absolute, descend,
-            metadata):
+    def _read_mozbuild(self, path, config, read_tiers, filesystem_absolute,
+            descend, metadata):
         path = mozpath.normpath(path)
         log(self._log, logging.DEBUG, 'read_mozbuild', {'path': path},
             'Reading file: {path}')
         if path in self._read_files:
             log(self._log, logging.WARNING, 'read_already', {'path': path},
                 'File already read. Skipping: {path}')
         time_start = time.time()
-        sandbox = MozbuildSandbox(self.config, path, metadata=metadata)
+        sandbox = MozbuildSandbox(config, path, metadata=metadata)
         sandbox.exec_file(path, filesystem_absolute=filesystem_absolute)
         sandbox.execution_time = time.time() - time_start
         if self._sandbox_post_eval_cb:
         var = metadata.get('var', None)
         forbidden = {
@@ -742,17 +742,17 @@ class BuildReader(object):
                     'The %s variable%s not allowed in such directories.'
                     % (sandbox['RELATIVEDIR'], var, metadata['parent'],
                        ' and '.join(', '.join(matches).rsplit(', ', 1)),
                        's are' if len(matches) > 1 else ' is'))
         # We first collect directories populated in variables.
         dir_vars = ['DIRS', 'PARALLEL_DIRS', 'TOOL_DIRS']
-        if self.config.substs.get('ENABLE_TESTS', False) == '1':
+        if sandbox.config.substs.get('ENABLE_TESTS', False) == '1':
             dir_vars.extend(['TEST_DIRS', 'TEST_TOOL_DIRS'])
         dirs = [(v, sandbox[v]) for v in dir_vars if v in sandbox]
         curdir = mozpath.dirname(path)
         gyp_sandboxes = []
         for target_dir in sandbox['GYP_DIRS']:
@@ -770,17 +770,17 @@ class BuildReader(object):
             from .gyp_reader import read_from_gyp
             non_unified_sources = set()
             for s in gyp_dir.non_unified_sources:
                 source = mozpath.normpath(mozpath.join(curdir, s))
                 if not os.path.exists(source):
                     raise SandboxValidationError('Cannot find %s referenced '
                         'from %s' % (source, path))
-            for gyp_sandbox in read_from_gyp(self.config,
+            for gyp_sandbox in read_from_gyp(sandbox.config,
                                              mozpath.join(curdir, gyp_dir.input),
                                              non_unified_sources = non_unified_sources):
@@ -842,21 +842,22 @@ class BuildReader(object):
         for relpath, child_metadata in recurse_info.items():
             child_path = mozpath.join(curdir, relpath, 'moz.build')
             # Ensure we don't break out of the topsrcdir. We don't do realpath
             # because it isn't necessary. If there are symlinks in the srcdir,
             # that's not our problem. We're not a hosted application: we don't
             # need to worry about security too much.
             child_path = mozpath.normpath(child_path)
-            if not is_read_allowed(child_path, self.config):
+            if not is_read_allowed(child_path, sandbox.config):
                 raise SandboxValidationError(
                     'Attempting to process file outside of allowed paths: %s' %
             if not descend:
-            for res in self.read_mozbuild(child_path, read_tiers=False,
-                filesystem_absolute=True, metadata=child_metadata):
+            for res in self.read_mozbuild(child_path, sandbox.config,
+                read_tiers=False, filesystem_absolute=True,
+                metadata=child_metadata):
                 yield res
--- a/python/mozbuild/mozbuild/test/frontend/test_reader.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py
@@ -52,17 +52,17 @@ class TestBuildReader(unittest.TestCase)
         self.assertEqual(len(sandboxes), 4)
     def test_dirs_traversal_no_descend(self):
         reader = self.reader('traversal-simple')
         path = mozpath.join(reader.topsrcdir, 'moz.build')
-        sandboxes = list(reader.read_mozbuild(path,
+        sandboxes = list(reader.read_mozbuild(path, reader.config,
             filesystem_absolute=True, descend=False))
         self.assertEqual(len(sandboxes), 1)
     def test_dirs_traversal_all_variables(self):
         reader = self.reader('traversal-all-vars', enable_tests=True)
         sandboxes = list(reader.read_topsrcdir())