Bug 1132771 - Support reading relevant moz.build files; r=glandium
☠☠ backed out by ce21e7a57cf8 ☠ ☠
authorGregory Szorc <gps@mozilla.com>
Thu, 26 Feb 2015 09:44:55 -0800
changeset 249544 6c44edc8208a54a9d5d830266cded7b409a776e1
parent 249543 ed135df395751194bf379584a4d210f14ac849b4
child 249545 91d34d3107faa777264cdcc9c89456bf4c289466
push id7860
push userjlund@mozilla.com
push dateMon, 30 Mar 2015 18:46:02 +0000
treeherdermozilla-aurora@8ac636cd51f3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1132771
milestone39.0a1
Bug 1132771 - Support reading relevant moz.build files; r=glandium Building on top of the API to retrieve relevant moz.build files for a given path, we introduce a moz.build reading API that reads all moz.build files relevant to a given set of paths. We plan to use this new API to read metadata from moz.build files relevant to a set of files. This patch changes the generator behavior of read_mozbuild to emit the main context before any processing occurs. This allows downstream consumers to manipulate state of the context before things like directory processing occurs. We utilize this capability in the new reading API to forcefully declare the directory traversal order for processed moz.build files, overriding DIRS and similar variables. Since variable exporting doesn't work reliably in this new traversal mode, variable exporting no-ops when this mode is activated.
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/test/frontend/test_reader.py
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -24,17 +24,20 @@ import logging
 import os
 import sys
 import textwrap
 import time
 import tokenize
 import traceback
 import types
 
-from collections import OrderedDict
+from collections import (
+    defaultdict,
+    OrderedDict,
+)
 from io import StringIO
 
 from mozbuild.util import (
     memoize,
     ReadOnlyDefaultDict,
     ReadOnlyDict,
 )
 
@@ -58,16 +61,17 @@ from .sandbox import (
 from .context import (
     Context,
     ContextDerivedValue,
     FUNCTIONS,
     VARIABLES,
     DEPRECATION_HINTS,
     SPECIAL_VARIABLES,
     SUBCONTEXTS,
+    SubContext,
     TemplateContext,
 )
 
 if sys.version_info.major == 2:
     text_type = unicode
     type_type = types.TypeType
 else:
     text_type = str
@@ -980,16 +984,21 @@ class BuildReader(object):
             config.topobjdir = topobjdir
             config.external_source_dir = None
 
         context = Context(VARIABLES, config)
         sandbox = MozbuildSandbox(context, metadata=metadata)
         sandbox.exec_file(path)
         context.execution_time = time.time() - time_start
 
+        # Yield main context before doing any processing. This gives immediate
+        # consumers an opportunity to change state before our remaining
+        # processing is performed.
+        yield context
+
         # We first collect directories populated in variables.
         dir_vars = ['DIRS']
 
         if context.config.substs.get('ENABLE_TESTS', False) == '1':
             dir_vars.append('TEST_DIRS')
 
         dirs = [(v, context[v]) for v in dir_vars if v in context]
 
@@ -1024,18 +1033,16 @@ class BuildReader(object):
                                              non_unified_sources = non_unified_sources):
                 gyp_context.update(gyp_dir.sandbox_vars)
                 gyp_contexts.append(gyp_context)
 
         for gyp_context in gyp_contexts:
             context['DIRS'].append(mozpath.relpath(gyp_context.objdir, context.objdir))
             sandbox.subcontexts.append(gyp_context)
 
-        yield context
-
         for subcontext in sandbox.subcontexts:
             yield subcontext
 
         # Traverse into referenced files.
 
         # It's very tempting to use a set here. Unfortunately, the recursive
         # make backend needs order preserved. Once we autogenerate all backend
         # files, we should be able to convert this to a set.
@@ -1116,8 +1123,74 @@ class BuildReader(object):
                 if not mozpath.basedir(path, [root]):
                     raise Exception('Path outside topsrcdir: %s' % path)
                 path = mozpath.relpath(path, root)
 
             result[path] = [p for p in itermozbuild(path)
                               if exists(mozpath.join(root, p))]
 
         return result
+
+    def read_relevant_mozbuilds(self, paths):
+        """Read and process moz.build files relevant for a set of paths.
+
+        For an iterable of relative-to-root filesystem paths ``paths``,
+        find all moz.build files that may apply to them based on filesystem
+        hierarchy and read those moz.build files.
+
+        The return value is a 2-tuple. The first item is a dict mapping each
+        input filesystem path to a list of Context instances that are relevant
+        to that path. The second item is a list of all Context instances. Each
+        Context instance is in both data structures.
+        """
+        relevants = self._find_relevant_mozbuilds(paths)
+
+        topsrcdir = self.config.topsrcdir
+
+        # Source moz.build file to directories to traverse.
+        dirs = defaultdict(set)
+        # Relevant path to absolute paths of relevant contexts.
+        path_mozbuilds = {}
+
+        # There is room to improve this code (and the code in
+        # _find_relevant_mozbuilds) to better handle multiple files in the same
+        # directory. Bug 1136966 tracks.
+        for path, mbpaths in relevants.items():
+            path_mozbuilds[path] = [mozpath.join(topsrcdir, p) for p in mbpaths]
+
+            for i, mbpath in enumerate(mbpaths[0:-1]):
+                source_dir = mozpath.dirname(mbpath)
+                target_dir = mozpath.dirname(mbpaths[i + 1])
+
+                d = mozpath.normpath(mozpath.join(topsrcdir, mbpath))
+                dirs[d].add(mozpath.relpath(target_dir, source_dir))
+
+        # Exporting doesn't work reliably in tree traversal mode. Override
+        # the function to no-op.
+        functions = dict(FUNCTIONS)
+        def export(sandbox):
+            return lambda varname: None
+        functions['export'] = tuple([export] + list(FUNCTIONS['export'][1:]))
+
+        metadata = {
+            'functions': functions,
+        }
+
+        contexts = defaultdict(list)
+        all_contexts = []
+        for context in self.read_mozbuild(mozpath.join(topsrcdir, 'moz.build'),
+                                          self.config, metadata=metadata):
+            # Explicitly set directory traversal variables to override default
+            # traversal rules.
+            if not isinstance(context, SubContext):
+                for v in ('DIRS', 'GYP_DIRS', 'TEST_DIRS'):
+                    context[v][:] = []
+
+                context['DIRS'] = sorted(dirs[context.main_path])
+
+            contexts[context.main_path].append(context)
+            all_contexts.append(context)
+
+        result = {}
+        for path, paths in path_mozbuilds.items():
+            result[path] = reduce(lambda x, y: x + y, (contexts[p] for p in paths), [])
+
+        return result, all_contexts
--- a/python/mozbuild/mozbuild/test/frontend/test_reader.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py
@@ -293,11 +293,26 @@ class TestBuildReader(unittest.TestCase)
         # Different root directories.
         paths = reader._find_relevant_mozbuilds(['d1/file', 'd2/file', 'file'])
         self.assertEqual(paths, {
             'file': ['moz.build'],
             'd1/file': ['moz.build', 'd1/moz.build'],
             'd2/file': ['moz.build', 'd2/moz.build'],
         })
 
+    def test_read_relevant_mozbuilds(self):
+        reader = self.reader('reader-relevant-mozbuild')
+
+        paths, contexts = reader.read_relevant_mozbuilds(['d1/every-level/a/file',
+            'd1/every-level/b/file', 'd2/file'])
+        self.assertEqual(len(paths), 3)
+        self.assertEqual(len(contexts), 6)
+
+        self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/a/file']],
+            ['', 'd1', 'd1/every-level', 'd1/every-level/a'])
+        self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/b/file']],
+            ['', 'd1', 'd1/every-level', 'd1/every-level/b'])
+        self.assertEqual([ctx.relsrcdir for ctx in paths['d2/file']],
+            ['', 'd2'])
+
 
 if __name__ == '__main__':
     main()