Bug 1132771 - Support reading relevant moz.build files draft
authorGregory Szorc <gps@mozilla.com>
Tue, 24 Feb 2015 12:36:54 -0800
changeset 245740 097cb6af15b54f89d9280147a289abf5eeba6154
parent 245739 ad85cd50819d61d8a0c0a8eb3b88aa4ff2123031
child 245741 deda19f28f5e82863604177fd2ec6a53d9f8ab87
push id795
push usergszorc@mozilla.com
push dateWed, 25 Feb 2015 00:26:50 +0000
bugs1132771
milestone39.0a1
Bug 1132771 - Support reading relevant moz.build files Building on top of the API to retrieve relevant moz.build files for a given path, we introduce a moz.build reading API that reads all moz.build files relevant to a given set of paths. We plan to use this new API to read metadata from moz.build files relevant to a set of files. This patch changes the generator behavior of read_mozbuild to emit the main context before any processing occurs. This allows downstream consumers to manipulate state of the context before things like directory processing occurs. We utilize this capability in the new reading API to forcefully declare the directory traversal order for processed moz.build files, overriding DIRS and similar variables.
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/test/frontend/test_reader.py
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -24,17 +24,20 @@ import logging
 import os
 import sys
 import textwrap
 import time
 import tokenize
 import traceback
 import types
 
-from collections import OrderedDict
+from collections import (
+    defaultdict,
+    OrderedDict,
+)
 from io import StringIO
 
 from mozbuild.util import (
     memoize,
     ReadOnlyDefaultDict,
     ReadOnlyDict,
 )
 
@@ -983,16 +986,21 @@ class BuildReader(object):
             config.topobjdir = topobjdir
             config.external_source_dir = None
 
         context = MainContext(config)
         sandbox = MozbuildSandbox(context, metadata=metadata)
         sandbox.exec_file(path)
         context.execution_time = time.time() - time_start
 
+        # Yield main context before doing any processing. This gives immediate
+        # consumers an opportunity to change state before our remaining
+        # processing is performed.
+        yield context
+
         # We first collect directories populated in variables.
         dir_vars = ['DIRS']
 
         if context.config.substs.get('ENABLE_TESTS', False) == '1':
             dir_vars.append('TEST_DIRS')
 
         dirs = [(v, context[v]) for v in dir_vars if v in context]
 
@@ -1027,18 +1035,16 @@ class BuildReader(object):
                                              non_unified_sources = non_unified_sources):
                 gyp_context.update(gyp_dir.sandbox_vars)
                 gyp_contexts.append(gyp_context)
 
         for gyp_context in gyp_contexts:
             context['DIRS'].append(mozpath.relpath(gyp_context.objdir, context.objdir))
             sandbox.subcontexts.append(gyp_context)
 
-        yield context
-
         for subcontext in sandbox.subcontexts:
             yield subcontext
 
         # Traverse into referenced files.
 
         # It's very tempting to use a set here. Unfortunately, the recursive
         # make backend needs order preserved. Once we autogenerate all backend
         # files, we should be able to convert this to a set.
@@ -1120,8 +1126,60 @@ class BuildReader(object):
                 if not mozpath.basedir(path, [root]):
                     raise Exception('Path outside topsrcdir: %s' % path)
                 path = mozpath.relpath(path, root)
 
             result[path] = [p for p in itermozbuild(path)
                               if exists(mozpath.join(root, p))]
 
         return result
+
+    def read_relevant_mozbuilds(self, paths):
+        """Read and process moz.build files relevant for a set of paths.
+
+        For an iterable of relative-to-root filesystem paths ``paths``,
+        find all moz.build files that may apply to them based on filesystem
+        hierarchy and read those moz.build files.
+
+        The return value is a 2-tuple. The first item is a dict mapping each
+        input filesystem path to a list of Context instances that are relevant
+        to that path. The second item is a list of all Context instances. Each
+        Context instance is in both data structures.
+        """
+        relevants = self._find_relevant_mozbuilds(paths)
+
+        topsrcdir = self.config.topsrcdir
+
+        # Source moz.build file to directories to traverse.
+        dirs = defaultdict(set)
+        # Relevant path to absolute paths of relevant contexts.
+        path_mozbuilds = {}
+
+        for path, mbpaths in relevants.items():
+            path_mozbuilds[path] = [mozpath.join(topsrcdir, p) for p in mbpaths]
+
+            for i, mbpath in enumerate(mbpaths[0:-1]):
+                source_dir = mozpath.dirname(mbpath)
+                target_dir = mozpath.dirname(mbpaths[i + 1])
+
+                d = mozpath.normpath(mozpath.join(topsrcdir, mbpath))
+                dirs[d].add(mozpath.relpath(target_dir, source_dir))
+
+        contexts = defaultdict(list)
+        all_contexts = []
+        for context in self.read_mozbuild(mozpath.join(topsrcdir, 'moz.build'),
+                                          self.config, read_tiers=False):
+            # Explicitly set directory traversal variables to override default
+            # traversal rules.
+            if isinstance(context, MainContext):
+                for v in ('DIRS', 'GYP_DIRS', 'TEST_DIRS'):
+                    context[v][:] = []
+
+                context['DIRS'] = sorted(dirs[context.main_path])
+
+            contexts[context.main_path].append(context)
+            all_contexts.append(context)
+
+        result = {}
+        for path, paths in path_mozbuilds.items():
+            result[path] = reduce(lambda x, y: x + y, (contexts[p] for p in paths), [])
+
+        return result, all_contexts
--- a/python/mozbuild/mozbuild/test/frontend/test_reader.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py
@@ -293,11 +293,26 @@ class TestBuildReader(unittest.TestCase)
         # Different root directories.
         paths = reader._find_relevant_mozbuilds(['d1/file', 'd2/file', 'file'])
         self.assertEqual(paths, {
             'file': ['moz.build'],
             'd1/file': ['moz.build', 'd1/moz.build'],
             'd2/file': ['moz.build', 'd2/moz.build'],
         })
 
+    def test_read_relevant_mozbuilds(self):
+        reader = self.reader('reader-relevant-mozbuild')
+
+        paths, contexts = reader.read_relevant_mozbuilds(['d1/every-level/a/file',
+            'd1/every-level/b/file', 'd2/file'])
+        self.assertEqual(len(paths), 3)
+        self.assertEqual(len(contexts), 6)
+
+        self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/a/file']],
+            ['', 'd1', 'd1/every-level', 'd1/every-level/a'])
+        self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/b/file']],
+            ['', 'd1', 'd1/every-level', 'd1/every-level/b'])
+        self.assertEqual([ctx.relsrcdir for ctx in paths['d2/file']],
+            ['', 'd2'])
+
 
 if __name__ == '__main__':
     main()