Bug 1249091 - Add IMPACTED_TASKS to moz.build Files metadata; r?glandium draft
authorGregory Szorc <gps@mozilla.com>
Wed, 17 Feb 2016 13:30:32 -0800
changeset 331693 91b09cb2bbae5560ed8ab5a906d4f025b8892ff9
parent 331658 9c570cd3eaa13e96e739aa14ee1564d37b2f6eff
child 331694 c559632cdaa2b24252bfc0380c7ee5ad0bc1d2e8
push id11040
push usergszorc@mozilla.com
push dateWed, 17 Feb 2016 22:51:37 +0000
reviewersglandium
bugs1249091, 1245953
milestone47.0a1
Bug 1249091 - Add IMPACTED_TASKS to moz.build Files metadata; r?glandium Bug 1245953 added support for limiting TaskCluster task scheduling to when certain files change. This was implemented as a list of mozpath patterns in TaskCluster's YAML files. A drawback of that approach is YAML files in testing/taskcluster list paths from all over the tree. This is prone to getting out of sync and can be a pain to update. moz.build files already provide a mechanism for associating metadata with files. And TaskCluster's scheduling code runs on a full source code checkout and it knows which files have changed. Putting all that together means it is possible for TaskCluster to query moz.build files for the set of tasks that are relevant to a file. This commit introduces the IMPACTED_TASKS Files metadata variable for declaring which TaskCluster tasks are impacted by files [and should be scheduled]. A subsequent commit will teach the TaskCluster code to query this metadata. MozReview-Commit-ID: IOSu9kMcSvi
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/mach_commands.py
python/mozbuild/mozbuild/test/frontend/test_context.py
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -651,16 +651,17 @@ class Files(SubContext):
             further updates to a variable.
 
             When ``FINAL`` is set, the value of all variables defined in this
             context are marked as frozen and all subsequent writes to them
             are ignored during metadata reading.
 
             See :ref:`mozbuild_files_metadata_finalizing` for more info.
             """),
+
         'IMPACTED_TESTS': (DependentTestsEntry, list,
             """File patterns, tags, and flavors for tests relevant to these files.
 
             Maps source files to the tests potentially impacted by those files.
             Tests can be specified by file pattern, tag, or flavor.
 
             For example:
 
@@ -699,16 +700,26 @@ class Files(SubContext):
             with Files('dom/base/nsGlobalWindow.cpp'):
                 IMPACTED_TESTS.flavors += [
                     'mochitest',
                 ]
 
             Would suggest that nsGlobalWindow.cpp is potentially relevant to
             any plain mochitest.
             """),
+
+        'IMPACTED_TASKS': (StrictOrderingOnAppendList, list,
+            """TaskCluster tasks that should be executed when files change.
+
+            Not all automation tasks need to run all the time. Many automation
+            tasks only need to run after certain files change.
+
+            This variable stores a list of Task Cluster task names that are
+            impacted and should be executed when certain files change.
+            """),
     }
 
     def __init__(self, parent, pattern=None):
         super(Files, self).__init__(parent)
         self.pattern = pattern
         self.finalized = set()
         self.test_files = set()
         self.test_tags = set()
@@ -733,16 +744,20 @@ class Files(SubContext):
             if k in self.finalized:
                 continue
 
             # Only finalize variables defined in this instance.
             if k == 'FINAL':
                 self.finalized |= set(other) - {'FINAL'}
                 continue
 
+            if k == 'IMPACTED_TASKS':
+                self[k] += sorted(v)
+                continue
+
             self[k] = v
 
         return self
 
     def asdict(self):
         """Return this instance as a dict with built-in data structures.
 
         Call this to obtain an object suitable for serializing.
@@ -769,37 +784,42 @@ class Files(SubContext):
         :py:func:`mozbuild.frontend.context.Files` instances passed in are
         thus the "collapsed" (``__iadd__``ed) results of all ``Files`` from all
         moz.build files relevant to a specific path, not individual ``Files``
         instances from a single moz.build file.
         """
         d = {}
 
         bug_components = Counter()
+        impacted_tasks = set()
 
         for f in files.values():
             bug_component = f.get('BUG_COMPONENT')
             if bug_component:
                 bug_components[bug_component] += 1
 
+            impacted_tasks |= set(f.get('IMPACTED_TASKS', []))
+
         d['bug_component_counts'] = []
         for c, count in bug_components.most_common():
             component = (c.product, c.component)
             d['bug_component_counts'].append((c, count))
 
             if 'recommended_bug_component' not in d:
                 d['recommended_bug_component'] = component
                 recommended_count = count
             elif count == recommended_count:
                 # Don't recommend a component if it doesn't have a clear lead.
                 d['recommended_bug_component'] = None
 
         # In case no bug components.
         d.setdefault('recommended_bug_component', None)
 
+        d['impacted_tasks'] = sorted(impacted_tasks)
+
         return d
 
 
 # This defines functions that create sub-contexts.
 #
 # Values are classes that are SubContexts. The class name will be turned into
 # a function that when called emits an instance of that class.
 #
--- a/python/mozbuild/mozbuild/frontend/mach_commands.py
+++ b/python/mozbuild/mozbuild/frontend/mach_commands.py
@@ -153,16 +153,34 @@ class MozbuildFileCommands(MachCommandBa
                     print('\tRelevant flavors:')
                     for p in m.test_flavors:
                         print('\t\t%s' % p)
 
         except InvalidPathException as e:
             print(e.message)
             return 1
 
+    @SubCommand('file-info', 'impacted-tasks',
+                'Show TaskCluster tasks impacted by changes to files')
+    @CommandArgument('-r', '--rev',
+                     help='Version control revision to look up info from')
+    @CommandArgument('paths', nargs='*',
+                     help='Paths whose data to query')
+    def file_info_impacted_tasks(self, paths, rev=None):
+        try:
+            for p, m in sorted(self._get_files_info(paths, rev=rev).items()):
+                tasks = m.get('IMPACTED_TASKS', [])
+                if not tasks:
+                    continue
+
+                relpath = mozpath.relpath(p, self.topsrcdir)
+                print('%s: %s' % (relpath, ', '.join(sorted(tasks))))
+        except InvalidPathException as e:
+            print(e.message)
+            return 1
 
     def _get_reader(self, finder):
         from mozbuild.frontend.reader import (
             BuildReader,
             EmptyConfig,
         )
 
         config = EmptyConfig(self.topsrcdir)
--- a/python/mozbuild/mozbuild/test/frontend/test_context.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_context.py
@@ -662,27 +662,29 @@ class TestTypedRecord(unittest.TestCase)
 class TestFiles(unittest.TestCase):
     def test_aggregate_empty(self):
         c = Context({})
 
         files = {'moz.build': Files(c, pattern='**')}
 
         self.assertEqual(Files.aggregate(files), {
             'bug_component_counts': [],
+            'impacted_tasks': [],
             'recommended_bug_component': None,
         })
 
     def test_single_bug_component(self):
         c = Context({})
         f = Files(c, pattern='**')
         f['BUG_COMPONENT'] = (u'Product1', u'Component1')
 
         files = {'moz.build': f}
         self.assertEqual(Files.aggregate(files), {
             'bug_component_counts': [((u'Product1', u'Component1'), 1)],
+            'impacted_tasks': [],
             'recommended_bug_component': (u'Product1', u'Component1'),
         })
 
     def test_multiple_bug_components(self):
         c = Context({})
         f1 = Files(c, pattern='**')
         f1['BUG_COMPONENT'] = (u'Product1', u'Component1')
 
@@ -690,16 +692,17 @@ class TestFiles(unittest.TestCase):
         f2['BUG_COMPONENT'] = (u'Product2', u'Component2')
 
         files = {'a': f1, 'b': f2, 'c': f1}
         self.assertEqual(Files.aggregate(files), {
             'bug_component_counts': [
                 ((u'Product1', u'Component1'), 2),
                 ((u'Product2', u'Component2'), 1),
             ],
+            'impacted_tasks': [],
             'recommended_bug_component': (u'Product1', u'Component1'),
         })
 
     def test_no_recommended_bug_component(self):
         """If there is no clear count winner, we don't recommend a bug component."""
         c = Context({})
         f1 = Files(c, pattern='**')
         f1['BUG_COMPONENT'] = (u'Product1', u'Component1')
@@ -708,14 +711,28 @@ class TestFiles(unittest.TestCase):
         f2['BUG_COMPONENT'] = (u'Product2', u'Component2')
 
         files = {'a': f1, 'b': f2}
         self.assertEqual(Files.aggregate(files), {
             'bug_component_counts': [
                 ((u'Product1', u'Component1'), 1),
                 ((u'Product2', u'Component2'), 1),
             ],
+            'impacted_tasks': [],
             'recommended_bug_component': None,
         })
 
+    def test_impacted_tasks_aggregate(self):
+        c = Context({})
+        f1 = Files(c, pattern='**')
+        f1['IMPACTED_TASKS'] += ['task1', 'task2']
+        f2  = Files(c, pattern='**')
+        f2['IMPACTED_TASKS'] += ['task0', 'task3']
+
+        files = {'a': f1, 'b': f2}
+        self.assertEqual(Files.aggregate(files), {
+            'bug_component_counts': [],
+            'impacted_tasks': ['task0', 'task1', 'task2', 'task3'],
+            'recommended_bug_component': None,
+        })
 
 if __name__ == '__main__':
     main()