Bug 1132771 - Support explicit directory traversal of moz.build files draft
authorGregory Szorc <gps@mozilla.com>
Thu, 12 Feb 2015 18:11:21 -0800
changeset 243050 19c27d212882846885131ad0729294114cebe5b4
parent 243049 0fbc158c1aa0233d087c8ea270a3773838852a9d
child 243051 cd9997e4f538b5882e65beb19d3418584136df3a
push id698
push usergszorc@mozilla.com
push dateMon, 16 Feb 2015 20:34:37 +0000
bugs1132771
milestone38.0a1
Bug 1132771 - Support explicit directory traversal of moz.build files This patch teaches moz.build files how to traverse/descend with an arbitrary directory mapping, ignoring any traversal values embedded within *DIRS files. Using this new behavior is an API to read moz.build files relevant for a set of input filenames. We eventually plan to use this new API to read metadata from moz.build files relevant to a set of files, possibly just one file. There are still improvements to this implementation. While not tested, I'm almost entirely confident that traversal on mozilla-central will fail. Also, we will likely want to use a dummy config object so we can traverse without having build context. We should ideally be able to traverse on a fresh source checkout.
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/test/frontend/test_reader.py
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -57,16 +57,17 @@ from .sandbox import (
 
 from .context import (
     Context,
     ContextDerivedValue,
     FUNCTIONS,
     VARIABLES,
     DEPRECATION_HINTS,
     SPECIAL_VARIABLES,
+    SourcePath,
     TemplateContext,
 )
 
 if sys.version_info.major == 2:
     text_type = unicode
     type_type = types.TypeType
 else:
     text_type = str
@@ -869,17 +870,17 @@ class BuildReader(object):
 
             tree = ast.parse(source, full)
             Visitor().visit(tree)
 
             for name, key, value in assignments:
                 yield p, name, key, value
 
     def read_mozbuild(self, path, config, read_tiers=False, descend=True,
-                      read_gyp=True, metadata={}):
+                      read_gyp=True, explicit_dirs=None, metadata={}):
         """Read and process a mozbuild file, descending into children.
 
         This starts with a single mozbuild file, executes it, and descends into
         other referenced files per our traversal logic.
 
         The traversal logic is to iterate over the *DIRS variables, treating
         each element as a relative directory path. For each encountered
         directory, we will open the moz.build file located in that
@@ -890,27 +891,37 @@ class BuildReader(object):
         traversal as well.
 
         If descend is True (the default), we will descend into child
         directories and files per variable values.
 
         If read_gyp is True (the default), we will process GYP files referenced
         by moz.build files.
 
+        explicit_dirs can be used to control an explicit directory traversal
+        sequence. The default behavior is for *DIRS variables in evaluated
+        moz.build to control what gets executed next. When this argument is
+        defined as a dictionary, keys in the dictionary corresponding to the
+        path of the evaluated moz.build file and values are an iterable of
+        directories to evaluate next. Keys should be absolute paths. Values
+        are relative from the key paths. This effectively replaces DIRS
+        values after sandbox execution.
+
         Arbitrary metadata in the form of a dict can be passed into this
         function. This feature is intended to facilitate the build reader
         injecting state and annotations into moz.build files that is
         independent of the sandbox's execution context.
 
         Traversal is performed depth first (for no particular reason).
         """
         self._execution_stack.append(path)
         try:
             for s in self._read_mozbuild(path, config, read_tiers=read_tiers,
-                descend=descend, read_gyp=read_gyp, metadata=metadata):
+                descend=descend, read_gyp=read_gyp,
+                explicit_dirs=explicit_dirs, metadata=metadata):
                 yield s
 
         except BuildReaderError as bre:
             raise bre
 
         except SandboxCalledError as sce:
             raise BuildReaderError(list(self._execution_stack),
                 sys.exc_info()[2], sandbox_called_error=sce)
@@ -927,17 +938,17 @@ class BuildReader(object):
             raise BuildReaderError(list(self._execution_stack),
                 sys.exc_info()[2], validation_error=ve)
 
         except Exception as e:
             raise BuildReaderError(list(self._execution_stack),
                 sys.exc_info()[2], other_error=e)
 
     def _read_mozbuild(self, path, config, read_tiers, descend, read_gyp,
-                       metadata):
+                       explicit_dirs, metadata):
         path = mozpath.normpath(path)
         log(self._log, logging.DEBUG, 'read_mozbuild', {'path': path},
             'Reading file: {path}')
 
         if path in self._read_files:
             log(self._log, logging.WARNING, 'read_already', {'path': path},
                 'File already read. Skipping: {path}')
             return
@@ -1022,16 +1033,20 @@ class BuildReader(object):
                 context['DIRS'].append(mozpath.relpath(gyp_context.objdir, context.objdir))
 
         yield context
 
         for gyp_context in gyp_contexts:
             yield gyp_context
 
         # Traverse into referenced files.
+        if explicit_dirs:
+            reldirs = explicit_dirs.get(context.main_path, [])
+            # Value of variable shouldn't matter.
+            dirs = [('EXPLICIT_DIRS', [SourcePath(context, d) for d in reldirs])]
 
         # It's very tempting to use a set here. Unfortunately, the recursive
         # make backend needs order preserved. Once we autogenerate all backend
         # files, we should be able to convert this to a set.
         recurse_info = OrderedDict()
         for var, var_dirs in dirs:
             for d in var_dirs:
                 if d in recurse_info:
@@ -1058,17 +1073,19 @@ class BuildReader(object):
                 raise SandboxValidationError(
                     'Attempting to process file outside of allowed paths: %s' %
                         child_path, context)
 
             if not descend:
                 continue
 
             for res in self.read_mozbuild(child_path, context.config,
-                read_tiers=False, read_gyp=read_gyp, metadata=child_metadata):
+                read_tiers=False, read_gyp=read_gyp,
+                explicit_dirs=explicit_dirs,
+                metadata=child_metadata):
                 yield res
 
         self._execution_stack.pop()
 
     def _find_relevant_mozbuilds(self, paths):
         """Given a set of filesystem paths, find all relevant moz.build files.
 
         We assume that a moz.build file in a directory ancestry of a given path
@@ -1119,8 +1136,54 @@ class BuildReader(object):
 
                 current_dir = mozpath.dirname(current_dir)
 
             key = mozpath.relpath(path, root)
             relevant[key] = [mozpath.relpath(p, root)
                              for p in reversed(current_relevant)]
 
         return relevant, {mozpath.relpath(p, root) for p in seen_mozbuild}
+
+    def read_relevant_mozbuilds(self, paths):
+        """Read and process moz.build files relevant for a set of paths.
+
+        For an iterable of relative-to-root filesystem paths ``paths``,
+        find all moz.build files that may apply to them based on filesystem
+        hierarchy and read those moz.build files.
+
+        The return value is a tuple of 2 dicts. The first dict maps each
+        input filesystem path to a list of Context instances that are relevant
+        to that path. The second dict is a list of all Context instances. Each
+        Context instance is in both dicts.
+        """
+        relevants, mozbuilds = self._find_relevant_mozbuilds(paths)
+
+        topsrcdir = self.config.topsrcdir
+
+        # Source moz.build file to directories to traverse.
+        dirs = {}
+        # Relevant path to absolute paths of relevant contexts.
+        path_mozbuilds = {}
+
+        for path, mbpaths in relevants.items():
+            path_mozbuilds[path] = [mozpath.join(topsrcdir, p) for p in mbpaths]
+
+            for i, mbpath in enumerate(mbpaths[0:-1]):
+                source_dir = mozpath.dirname(mbpath)
+                target_dir = mozpath.dirname(mbpaths[i + 1])
+
+                d = mozpath.normpath(mozpath.join(topsrcdir, mbpath))
+                t = dirs.setdefault(d, set())
+                t.add(mozpath.relpath(target_dir, source_dir))
+
+        contexts = {}
+        all_contexts = []
+        for context in self.read_mozbuild(mozpath.join(topsrcdir, 'moz.build'),
+                                          self.config, read_tiers=False,
+                                          read_gyp=False, explicit_dirs=dirs):
+            contexts[context.main_path] = context
+            all_contexts.append(context)
+
+        result = {}
+        for path, paths in path_mozbuilds.items():
+            result[path] = [contexts[p] for p in paths]
+
+        return result, all_contexts
--- a/python/mozbuild/mozbuild/test/frontend/test_reader.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py
@@ -321,11 +321,19 @@ class TestBuildReader(unittest.TestCase)
         paths, mozbuilds = reader._find_relevant_mozbuilds(['d1/file', 'd2/file', 'file'])
         self.assertEqual(paths, {
             'file': ['moz.build'],
             'd1/file': ['moz.build', 'd1/moz.build'],
             'd2/file': ['moz.build', 'd2/moz.build'],
         })
         self.assertEqual(mozbuilds, {'moz.build', 'd1/moz.build', 'd2/moz.build'})
 
+    def test_read_relevant_mozbuilds(self):
+        reader = self.reader('reader-relevant-mozbuild')
+
+        paths, context = reader.read_relevant_mozbuilds(['d1/every-level/a/file',
+            'd1/every-level/b/file', 'd2/file'])
+        self.assertEqual(len(paths), 3)
+        self.assertEqual(len(paths), 3)
+
 
 if __name__ == '__main__':
     main()