Bug 1132771 - Support reading relevant moz.build files; r=glandium
authorGregory Szorc <gps@mozilla.com>
Thu, 26 Feb 2015 09:44:55 -0800
changeset 231463 3a6dbe9707270d90f2e2ff3ceb9651646be86e8b
parent 231462 d5c17696d9d598dd0de47d4d1ca01217d070c62d
child 231464 a6b72bee55a8b7a2a0f7e4e6e8b8a3c5da7946b7
push id28352
push usercbook@mozilla.com
push dateTue, 03 Mar 2015 12:51:09 +0000
treeherdermozilla-central@e545f650c695 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1132771
milestone39.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1132771 - Support reading relevant moz.build files; r=glandium Building on top of the API to retrieve relevant moz.build files for a given path, we introduce a moz.build reading API that reads all moz.build files relevant to a given set of paths. We plan to use this new API to read metadata from moz.build files relevant to a set of files. This patch changes the generator behavior of read_mozbuild to emit the main context before any processing occurs. This allows downstream consumers to manipulate state of the context before things like directory processing occurs. We utilize this capability in the new reading API to forcefully declare the directory traversal order for processed moz.build files, overriding DIRS and similar variables. Since variable exporting doesn't work reliably in this new traversal mode, variable exporting no-ops when this mode is activated.
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/test/frontend/test_reader.py
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -24,17 +24,20 @@ import logging
 import os
 import sys
 import textwrap
 import time
 import tokenize
 import traceback
 import types
 
-from collections import OrderedDict
+from collections import (
+    defaultdict,
+    OrderedDict,
+)
 from io import StringIO
 
 from mozbuild.util import (
     memoize,
     ReadOnlyDefaultDict,
     ReadOnlyDict,
 )
 
@@ -58,16 +61,17 @@ from .sandbox import (
 from .context import (
     Context,
     ContextDerivedValue,
     FUNCTIONS,
     VARIABLES,
     DEPRECATION_HINTS,
     SPECIAL_VARIABLES,
     SUBCONTEXTS,
+    SubContext,
     TemplateContext,
 )
 
 if sys.version_info.major == 2:
     text_type = unicode
     type_type = types.TypeType
 else:
     text_type = str
@@ -980,16 +984,21 @@ class BuildReader(object):
             config.topobjdir = topobjdir
             config.external_source_dir = None
 
         context = Context(VARIABLES, config)
         sandbox = MozbuildSandbox(context, metadata=metadata)
         sandbox.exec_file(path)
         context.execution_time = time.time() - time_start
 
+        # Yield main context before doing any processing. This gives immediate
+        # consumers an opportunity to change state before our remaining
+        # processing is performed.
+        yield context
+
         # We first collect directories populated in variables.
         dir_vars = ['DIRS']
 
         if context.config.substs.get('ENABLE_TESTS', False) == '1':
             dir_vars.append('TEST_DIRS')
 
         dirs = [(v, context[v]) for v in dir_vars if v in context]
 
@@ -1024,18 +1033,16 @@ class BuildReader(object):
                                              non_unified_sources = non_unified_sources):
                 gyp_context.update(gyp_dir.sandbox_vars)
                 gyp_contexts.append(gyp_context)
 
         for gyp_context in gyp_contexts:
             context['DIRS'].append(mozpath.relpath(gyp_context.objdir, context.objdir))
             sandbox.subcontexts.append(gyp_context)
 
-        yield context
-
         for subcontext in sandbox.subcontexts:
             yield subcontext
 
         # Traverse into referenced files.
 
         # It's very tempting to use a set here. Unfortunately, the recursive
         # make backend needs order preserved. Once we autogenerate all backend
         # files, we should be able to convert this to a set.
@@ -1116,8 +1123,74 @@ class BuildReader(object):
                 if not mozpath.basedir(path, [root]):
                     raise Exception('Path outside topsrcdir: %s' % path)
                 path = mozpath.relpath(path, root)
 
             result[path] = [p for p in itermozbuild(path)
                               if exists(mozpath.join(root, p))]
 
         return result
+
+    def read_relevant_mozbuilds(self, paths):
+        """Read and process moz.build files relevant for a set of paths.
+
+        For an iterable of relative-to-root filesystem paths ``paths``,
+        find all moz.build files that may apply to them based on filesystem
+        hierarchy and read those moz.build files.
+
+        The return value is a 2-tuple. The first item is a dict mapping each
+        input filesystem path to a list of Context instances that are relevant
+        to that path. The second item is a list of all Context instances. Each
+        Context instance is in both data structures.
+        """
+        relevants = self._find_relevant_mozbuilds(paths)
+
+        topsrcdir = self.config.topsrcdir
+
+        # Source moz.build file to directories to traverse.
+        dirs = defaultdict(set)
+        # Relevant path to absolute paths of relevant contexts.
+        path_mozbuilds = {}
+
+        # There is room to improve this code (and the code in
+        # _find_relevant_mozbuilds) to better handle multiple files in the same
+        # directory. Bug 1136966 tracks.
+        for path, mbpaths in relevants.items():
+            path_mozbuilds[path] = [mozpath.join(topsrcdir, p) for p in mbpaths]
+
+            for i, mbpath in enumerate(mbpaths[0:-1]):
+                source_dir = mozpath.dirname(mbpath)
+                target_dir = mozpath.dirname(mbpaths[i + 1])
+
+                d = mozpath.normpath(mozpath.join(topsrcdir, mbpath))
+                dirs[d].add(mozpath.relpath(target_dir, source_dir))
+
+        # Exporting doesn't work reliably in tree traversal mode. Override
+        # the function to no-op.
+        functions = dict(FUNCTIONS)
+        def export(sandbox):
+            return lambda varname: None
+        functions['export'] = tuple([export] + list(FUNCTIONS['export'][1:]))
+
+        metadata = {
+            'functions': functions,
+        }
+
+        contexts = defaultdict(list)
+        all_contexts = []
+        for context in self.read_mozbuild(mozpath.join(topsrcdir, 'moz.build'),
+                                          self.config, metadata=metadata):
+            # Explicitly set directory traversal variables to override default
+            # traversal rules.
+            if not isinstance(context, SubContext):
+                for v in ('DIRS', 'GYP_DIRS', 'TEST_DIRS'):
+                    context[v][:] = []
+
+                context['DIRS'] = sorted(dirs[context.main_path])
+
+            contexts[context.main_path].append(context)
+            all_contexts.append(context)
+
+        result = {}
+        for path, paths in path_mozbuilds.items():
+            result[path] = reduce(lambda x, y: x + y, (contexts[p] for p in paths), [])
+
+        return result, all_contexts
--- a/python/mozbuild/mozbuild/test/frontend/test_reader.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py
@@ -293,11 +293,26 @@ class TestBuildReader(unittest.TestCase)
         # Different root directories.
         paths = reader._find_relevant_mozbuilds(['d1/file', 'd2/file', 'file'])
         self.assertEqual(paths, {
             'file': ['moz.build'],
             'd1/file': ['moz.build', 'd1/moz.build'],
             'd2/file': ['moz.build', 'd2/moz.build'],
         })
 
+    def test_read_relevant_mozbuilds(self):
+        reader = self.reader('reader-relevant-mozbuild')
+
+        paths, contexts = reader.read_relevant_mozbuilds(['d1/every-level/a/file',
+            'd1/every-level/b/file', 'd2/file'])
+        self.assertEqual(len(paths), 3)
+        self.assertEqual(len(contexts), 6)
+
+        self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/a/file']],
+            ['', 'd1', 'd1/every-level', 'd1/every-level/a'])
+        self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/b/file']],
+            ['', 'd1', 'd1/every-level', 'd1/every-level/b'])
+        self.assertEqual([ctx.relsrcdir for ctx in paths['d2/file']],
+            ['', 'd2'])
+
 
 if __name__ == '__main__':
     main()