Bug 1132771 - Add FILES to moz.build with ability to define Bugzilla component draft
authorGregory Szorc <gps@mozilla.com>
Mon, 16 Feb 2015 12:29:56 -0800
changeset 243054 d9e060dde9a775d921c2bd137bc35aec2c40a935
parent 243053 cfe32922599a48121ec58255f51db351a6de1f80
child 505344 56d98197b4dbbe876e031ada1c0b748b1da7b80a
push id698
push usergszorc@mozilla.com
push dateMon, 16 Feb 2015 20:34:37 +0000
bugs1132771
milestone38.0a1
Bug 1132771 - Add FILES to moz.build with ability to define Bugzilla component Now that we can obtain relevant contexts from file paths, we introduce our first consumer of this API: the FILES variable. FILES is a dictionary of file matching patterns to metadata for matching files. e.g. you say "for all .cpp files in a directory, define the attribute X." We implement the "bug_component" attribute. This attribute is a 2-tuple (actually a named tuple) defining the Bugzilla product and component for files. There are no consumers yet. But we eventually want to enable things like "suggest a bug component for the patch I just wrote."
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/bad-assignment/moz.build
python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/different-matchers/moz.build
python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/final/moz.build
python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/final/subcomponent/moz.build
python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/moz.build
python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/simple/moz.build
python/mozbuild/mozbuild/test/frontend/data/files-metadata/moz.build
python/mozbuild/mozbuild/test/frontend/test_reader.py
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -13,25 +13,30 @@ It also defines the set of variables and
 If you are looking for the absolute authority on what moz.build files can
 contain, you've come to the right place.
 """
 
 from __future__ import unicode_literals
 
 import os
 
-from collections import OrderedDict
+from collections import (
+    namedtuple,
+    OrderedDict,
+)
 from contextlib import contextmanager
 from mozbuild.util import (
+    FlagsFactory,
     HierarchicalStringList,
     HierarchicalStringListWithFlagsFactory,
     KeyedDefaultDict,
     List,
     memoize,
     memoized_property,
+    OrderedDefaultDict,
     ReadOnlyKeyedDefaultDict,
     StrictOrderingOnAppendList,
     StrictOrderingOnAppendListWithFlagsFactory,
     TypedList,
 )
 import mozpack.path as mozpath
 from types import FunctionType
 from UserString import UserString
@@ -361,32 +366,111 @@ def ContextDerivedTypedList(type, base_c
                 def __new__(cls, obj):
                     return type(context, obj)
             self.TYPE = _Type
             super(_TypedList, self).__init__(iterable)
 
     return _TypedList
 
 
+BugzillaComponent = namedtuple('BugzillaComponent', ['product', 'component'])
+
+FilesFlagsBase = FlagsFactory({
+    'bug_component': (BugzillaComponent, lambda f, n, v: BugzillaComponent(*v)),
+    'final': set,
+})
+
+
+class FilesFlags(FilesFlagsBase):
+    """Represent flags on a FILES entry.
+
+    This class implements the ``+=`` operator so multiple instances can be
+    merged together. This represents the stacking of applicable flags on top
+    of each other from matched rules (see FILES documentation).
+    """
+    def __iadd__(self, other):
+        assert isinstance(other, FilesFlags)
+
+        for a in other.__slots__:
+            # Ignore updates to finalized flags.
+            if a in self.final:
+                continue
+
+            # Finalized flags should be unioned.
+            if a == 'final':
+                self.final |= other.final
+                continue
+
+            setattr(self, a, getattr(other, a))
+
+        return self
+
+
+class FilesVariable(OrderedDefaultDict):
+    """Represents the FILES variable."""
+    def __init__(self, *args, **kwargs):
+        super(FilesVariable, self).__init__(FilesFlags, *args, **kwargs)
+
+
 # This defines the set of mutable global variables.
 #
 # Each variable is a tuple of:
 #
 #   (storage_type, input_types, docs, tier)
 #
 # Tier says for which specific tier the variable has an effect.
 # Valid tiers are:
 # - 'export'
 # - 'libs': everything that is not built from C/C++/ObjC source and that has
 #      traditionally been in the libs tier.
 # - 'misc': like libs, but with parallel build. Eventually, everything that
 #      currently is in libs should move here.
 # A value of None means the variable has no direct effect on any tier.
 
 VARIABLES = {
+    'FILES': (FilesVariable, 'dict',
+        """Metadata attached to files or directories.
+
+        We often want to annotate files in the source checkout with additional
+        metadata, such as Bugzilla components that pertain to these files. This
+        variable is where we stick that metadata.
+
+        This variable is a special ordered dictionary. Keys in the dictionary
+        correspond to file matching patterns that are applied against the
+        current directory. e.g. ``foo.html`` will match the ``foo.html`` file.
+        ``*.jsm`` will match all ``.jsm`` files in the current directory.
+        ``**/*.cpp`` will match all ``.cpp`` files in child directories.
+
+        Values in the dictionary are special classes whose attributes define
+        metadata. The following attributes may be defined:
+
+        bug_component
+           A 2-tuple of unicode describing the Bugzilla product and component
+           that track issues with matched files.
+        final
+           A set of attribute names whose values to "freeze" so further updates
+           are ignored.
+
+           Some patterns should take precedence over others. While the order
+           matchings are defined in does dictate evaluation order, there are
+           some cases where a value should be set and frozen to prevent
+           overrides.
+
+           One use case is for files that are part of a logical group but exist
+           in many directories. e.g. ``Makefile.in`` files are part of the
+           build system, so ``**/Makefile.in`` should probably be defined at
+           the root directory. However, unless ``final`` is defined, wildcard
+           `*` and `**` file patterns would overrite this root value.
+
+        Keys in this dict will be processed in the order they are assigned.
+        For each file we are seeking information about, we will iterate over
+        keys and apply attribute values from filename patterns that match this
+        file. Last write wins.
+        """, None),
+
     # Variables controlling reading of other frontend files.
     'ANDROID_GENERATED_RESFILES': (StrictOrderingOnAppendList, list,
         """Android resource files generated as part of the build.
 
         This variable contains a list of files that are expected to be
         generated (often by preprocessing) into a 'res' directory as
         part of the build process, and subsequently merged into an APK
         file.
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -53,16 +53,17 @@ from .sandbox import (
     SandboxExecutionError,
     SandboxLoadError,
     Sandbox,
 )
 
 from .context import (
     Context,
     ContextDerivedValue,
+    FilesFlags,
     FUNCTIONS,
     VARIABLES,
     DEPRECATION_HINTS,
     SPECIAL_VARIABLES,
     SourcePath,
     TemplateContext,
 )
 
@@ -1188,8 +1189,52 @@ class BuildReader(object):
             contexts[context.main_path] = context
             all_contexts.append(context)
 
         result = {}
         for path, paths in path_mozbuilds.items():
             result[path] = [contexts[p] for p in paths]
 
         return result, all_contexts
+
+    def get_metadata_for_files(self, paths):
+        """Obtain metadata for a set of files.
+
+        Given a set of input paths, determine which moz.build files may
+        define metadata for them, evaluate those moz.build files, and
+        apply file metadata rules defined within to determine metadata
+        values for each file requested.
+
+        Essentially, for each input path:
+
+        1. Determine the set of moz.build files relevant to that file by
+           looking for moz.build files in ancestor directories.
+        2. Evaluate moz.build files starting with the most distant.
+        3. Iterate over entries in ``FILES`` in each moz.build context.
+        4. If the file pattern matches the file we're seeking info on,
+           apply attribute updates.
+        5. Return the most recent value of attributes.
+
+        In practice, moz.build files at the root of the tree are expected
+        to contain generic metadata definitions. As we get closer to leaves,
+        metadata becomes more specific and overwrites the parent metadata.
+        """
+        paths, _ = self.read_relevant_mozbuilds(paths)
+
+        r = {}
+
+        for path, ctxs in paths.items():
+            flags = FilesFlags()
+
+            for ctx in ctxs:
+                if 'FILES' not in ctx:
+                    continue
+
+                relpath = mozpath.relpath(path, ctx.relsrcdir)
+                for pattern, newflags in ctx['FILES'].items():
+                    if not mozpath.match(relpath, pattern):
+                        continue
+
+                    flags += newflags
+
+            r[path] = flags
+
+        return r
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/bad-assignment/moz.build
@@ -0,0 +1,1 @@
+FILES['*'].bug_component = 'bad value'
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/different-matchers/moz.build
@@ -0,0 +1,2 @@
+FILES['*.jsm'].bug_component = ('Firefox', 'JS')
+FILES['*.cpp'].bug_component = ('Firefox', 'C++')
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/final/moz.build
@@ -0,0 +1,2 @@
+FILES['**/Makefile.in'].bug_component = ('Core', 'Build Config')
+FILES['**/Makefile.in'].final.add('bug_component')
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/final/subcomponent/moz.build
@@ -0,0 +1,1 @@
+FILES['**'].bug_component = ('Another', 'Component')
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/moz.build
@@ -0,0 +1,1 @@
+FILES['**'].bug_component = ('default_product', 'default_component')
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/files-metadata/bug_component/simple/moz.build
@@ -0,0 +1,1 @@
+FILES['*'].bug_component = ('Core', 'Build Config')
new file mode 100644
--- a/python/mozbuild/mozbuild/test/frontend/test_reader.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py
@@ -336,11 +336,61 @@ class TestBuildReader(unittest.TestCase)
         self.assertEqual(len(paths), 3)
 
     def test_relevant_file_reading(self):
         mb = MozbuildObject.from_environment()
         config = mb.config_environment
         reader = BuildReader(config)
         paths, contexts = reader.read_relevant_mozbuilds(reader.all_mozbuild_paths())
 
+    def test_files_bad_bug_component(self):
+        reader = self.reader('files-metadata')
+
+        with self.assertRaises(BuildReaderError):
+            reader.get_metadata_for_files(['bug_component/bad-assignment/moz.build'])
+
+    def test_files_bug_component_simple(self):
+        reader = self.reader('files-metadata')
+
+        v = reader.get_metadata_for_files(['bug_component/simple/moz.build'])
+        self.assertEqual(len(v), 1)
+        flags = v['bug_component/simple/moz.build']
+        self.assertEqual(flags.bug_component.product, 'Core')
+        self.assertEqual(flags.bug_component.component, 'Build Config')
+
+    def test_files_bug_component_different_matchers(self):
+        reader = self.reader('files-metadata')
+
+        v = reader.get_metadata_for_files([
+            'bug_component/different-matchers/foo.jsm',
+            'bug_component/different-matchers/bar.cpp',
+            'bug_component/different-matchers/baz.misc'])
+        self.assertEqual(len(v), 3)
+
+        js_flags = v['bug_component/different-matchers/foo.jsm']
+        cpp_flags = v['bug_component/different-matchers/bar.cpp']
+        misc_flags = v['bug_component/different-matchers/baz.misc']
+
+        self.assertEqual(js_flags.bug_component, ('Firefox', 'JS'))
+        self.assertEqual(cpp_flags.bug_component, ('Firefox', 'C++'))
+        self.assertEqual(misc_flags.bug_component, ('default_product', 'default_component'))
+
+    def test_files_bug_component_final(self):
+        reader = self.reader('files-metadata')
+
+        v = reader.get_metadata_for_files([
+            'bug_component/final/foo',
+            'bug_component/final/Makefile.in',
+            'bug_component/final/subcomponent/Makefile.in',
+            'bug_component/final/subcomponent/bar'])
+
+        self.assertEqual(v['bug_component/final/foo'].bug_component,
+            ('default_product', 'default_component'))
+        self.assertEqual(v['bug_component/final/Makefile.in'].bug_component,
+            ('Core', 'Build Config'))
+        self.assertEqual(v['bug_component/final/subcomponent/Makefile.in'].bug_component,
+            ('Core', 'Build Config'))
+        self.assertEqual(v['bug_component/final/subcomponent/bar'].bug_component,
+            ('Another', 'Component'))
+
 
 if __name__ == '__main__':
     main()