Bug 1383880: add support for SCHEDULES in moz.build; r=gps
☠☠ backed out by 1c581da21a7d ☠ ☠
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 31 Jul 2017 20:44:56 +0000
changeset 428822 63ded86f8e0e02c50088d96f9ea5d74fbed55a2b
parent 428821 b53ff084c2d7968a1d9864d1343f2d9381fb652b
child 428823 046d705929f7a41e977eec19c8503afccdec7592
push id7761
push userjlund@mozilla.com
push dateFri, 15 Sep 2017 00:19:52 +0000
treeherdermozilla-beta@c38455951db4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1383880
milestone57.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1383880: add support for SCHEDULES in moz.build; r=gps MozReview-Commit-ID: 2pfLr0VTy2J
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/mach_commands.py
python/mozbuild/mozbuild/schedules.py
python/mozbuild/mozbuild/test/frontend/data/schedules/moz.build
python/mozbuild/mozbuild/test/frontend/data/schedules/subd/moz.build
python/mozbuild/mozbuild/test/frontend/test_reader.py
python/mozbuild/mozbuild/util.py
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -19,29 +19,32 @@ from __future__ import absolute_import, 
 import os
 
 from collections import (
     Counter,
     OrderedDict,
 )
 from mozbuild.util import (
     HierarchicalStringList,
+    ImmutableStrictOrderingOnAppendList,
     KeyedDefaultDict,
     List,
     ListWithAction,
     memoize,
     memoized_property,
     ReadOnlyKeyedDefaultDict,
     StrictOrderingOnAppendList,
     StrictOrderingOnAppendListWithAction,
     StrictOrderingOnAppendListWithFlagsFactory,
     TypedList,
     TypedNamedTuple,
 )
 
+from .. import schedules
+
 from ..testing import (
     all_test_flavors,
     read_manifestparser_manifest,
     read_reftest_manifest,
     read_wpt_manifest,
 )
 
 import mozpack.path as mozpath
@@ -554,16 +557,64 @@ def ContextDerivedTypedRecord(*fields):
             if name in self._fields and not isinstance(value, self._fields[name]):
                 value = self._fields[name](value)
             object.__setattr__(self, name, value)
 
     _TypedRecord._fields = dict(fields)
     return _TypedRecord
 
 
+class Schedules(object):
+    """Similar to a ContextDerivedTypedRecord, but with different behavior
+    for the properties:
+
+     * VAR.inclusive can only be appended to (+=), and can only contain values
+       from mozbuild.schedules.INCLUSIVE_COMPONENTS
+
+     * VAR.exclusive can only be assigned to (no +=), and can only contain
+       values from mozbuild.schedules.ALL_COMPONENTS
+    """
+    __slots__ = ('_exclusive', '_inclusive')
+
+    def __init__(self):
+        self._inclusive = TypedList(Enum(*schedules.INCLUSIVE_COMPONENTS))()
+        self._exclusive = ImmutableStrictOrderingOnAppendList(schedules.EXCLUSIVE_COMPONENTS)
+
+    # inclusive is mutable cannot be assigned to (+= only)
+    @property
+    def inclusive(self):
+        return self._inclusive
+
+    @inclusive.setter
+    def inclusive(self, value):
+        if value is not self._inclusive:
+            raise AttributeError("Cannot assign to this value - use += instead")
+        unexpected = [v for v in value if v not in schedules.INCLUSIVE_COMPONENTS]
+        if unexpected:
+            raise Exception("unexpected exclusive component(s) " + ', '.join(unexpected))
+
+    # exclusive is immuntable but can be set (= only)
+    @property
+    def exclusive(self):
+        return self._exclusive
+
+    @exclusive.setter
+    def exclusive(self, value):
+        if not isinstance(value, (tuple, list)):
+            raise Exception("expected a tuple or list")
+        unexpected = [v for v in value if v not in schedules.ALL_COMPONENTS]
+        if unexpected:
+            raise Exception("unexpected exclusive component(s) " + ', '.join(unexpected))
+        self._exclusive = ImmutableStrictOrderingOnAppendList(sorted(value))
+
+    # components provides a synthetic summary of all components
+    @property
+    def components(self):
+        return list(sorted(set(self._inclusive) | set(self._exclusive)))
+
 @memoize
 def ContextDerivedTypedHierarchicalStringList(type):
     """Specialized HierarchicalStringList for use with ContextDerivedValue
     types."""
     class _TypedListWithItems(ContextDerivedValue, HierarchicalStringList):
         __slots__ = ('_strings', '_children', '_context')
 
         def __init__(self, context):
@@ -624,16 +675,19 @@ OrderedSourceList = ContextDerivedTypedL
 OrderedTestFlavorList = TypedList(Enum(*all_test_flavors()),
                                   StrictOrderingOnAppendList)
 OrderedStringList = TypedList(unicode, StrictOrderingOnAppendList)
 DependentTestsEntry = ContextDerivedTypedRecord(('files', OrderedSourceList),
                                                 ('tags', OrderedStringList),
                                                 ('flavors', OrderedTestFlavorList))
 BugzillaComponent = TypedNamedTuple('BugzillaComponent',
                         [('product', unicode), ('component', unicode)])
+SchedulingComponents = ContextDerivedTypedRecord(
+        ('inclusive', TypedList(unicode, StrictOrderingOnAppendList)),
+        ('exclusive', TypedList(unicode, StrictOrderingOnAppendList)))
 
 
 class Files(SubContext):
     """Metadata attached to files.
 
     It is common to want to annotate files with metadata, such as which
     Bugzilla component tracks issues with certain files. This sub-context is
     where we stick that metadata.
@@ -742,16 +796,45 @@ class Files(SubContext):
             with Files('dom/base/nsGlobalWindow.cpp'):
                 IMPACTED_TESTS.flavors += [
                     'mochitest',
                 ]
 
             Would suggest that nsGlobalWindow.cpp is potentially relevant to
             any plain mochitest.
             """),
+        'SCHEDULES': (Schedules, list,
+            """Maps source files to the CI tasks that should be scheduled when
+            they change.  The tasks are grouped by named components, and those
+            names appear again in the taskgraph configuration
+            `($topsrcdir/taskgraph/).
+
+            Some components are "inclusive", meaning that changes to most files
+            do not schedule them, aside from those described in a Files
+            subcontext.  For example, py-lint tasks need not be scheduled for
+            most changes, but should be scheduled when any Python file changes.
+            Such components are named by appending to `SCHEDULES.inclusive`:
+
+            with Files('**.py'):
+                SCHEDULES.inclusive += ['py-lint']
+
+            Other components are 'exclusive', meaning that changes to most
+            files schedule them, but some files affect only one or two
+            components. For example, most files schedule builds and tests of
+            Firefox for Android, OS X, Windows, and Linux, but files under
+            `mobile/android/` affect Android builds and tests exclusively, so
+            builds for other operating systems are not needed.  Test suites
+            provide another example: most files schedule reftests, but changes
+            to reftest scripts need only schedule reftests and no other suites.
+
+            Exclusive components are named by setting `SCHEDULES.exclusive`:
+
+            with Files('mobile/android/**'):
+                SCHEDULES.exclusive = ['android']
+            """),
     }
 
     def __init__(self, parent, pattern=None):
         super(Files, self).__init__(parent)
         self.pattern = pattern
         self.finalized = set()
         self.test_files = set()
         self.test_tags = set()
--- a/python/mozbuild/mozbuild/frontend/mach_commands.py
+++ b/python/mozbuild/mozbuild/frontend/mach_commands.py
@@ -12,16 +12,17 @@ from mach.decorators import (
     CommandProvider,
     Command,
     SubCommand,
 )
 
 from mozbuild.base import MachCommandBase
 import mozpack.path as mozpath
 
+TOPSRCDIR = os.path.abspath(os.path.join(__file__, '../../../../../'))
 
 class InvalidPathException(Exception):
     """Represents an error due to an invalid path."""
 
 
 @CommandProvider
 class MozbuildFileCommands(MachCommandBase):
     @Command('mozbuild-reference', category='build-dev',
@@ -216,8 +217,28 @@ class MozbuildFileCommands(MachCommandBa
 
             for path, f in finder.find(p):
                 if path not in all_paths_set:
                     all_paths_set.add(path)
                     allpaths.append(path)
 
         reader = self._get_reader(finder=reader_finder)
         return reader.files_info(allpaths)
+
+
+    @SubCommand('file-info', 'schedules',
+                'Show the combined SCHEDULES for the files listed.')
+    @CommandArgument('paths', nargs='+',
+                     help='Paths whose data to query')
+    def file_info_schedules(self, paths):
+        """Show what is scheduled by the given files.
+
+        Given a requested set of files (which can be specified using
+        wildcards), print the total set of scheduled components.
+        """
+        from mozbuild.frontend.reader import EmptyConfig, BuildReader
+        config = EmptyConfig(TOPSRCDIR)
+        reader = BuildReader(config)
+        schedules = set()
+        for p, m in reader.files_info(paths).items():
+            schedules |= set(m['SCHEDULES'].components)
+
+        print(", ".join(schedules))
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/schedules.py
@@ -0,0 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Constants for SCHEDULES configuration in moz.build files and for
+skip-unless-schedules optimizations in task-graph generation.
+"""
+
+from __future__ import absolute_import, unicode_literals, print_function
+
+# TODO: ideally these lists could be specified in moz.build itself
+
+INCLUSIVE_COMPONENTS = [
+    'py-lint',
+    'js-lint',
+    'yaml-lint',
+]
+EXCLUSIVE_COMPONENTS = [
+    # os families
+    'android',
+    'linux',
+    'macosx',
+    'windows',
+]
+ALL_COMPONENTS = INCLUSIVE_COMPONENTS + EXCLUSIVE_COMPONENTS
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/schedules/moz.build
@@ -0,0 +1,11 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+with Files('*.win'):
+    SCHEDULES.exclusive = ['windows']
+
+with Files('*.osx'):
+    SCHEDULES.exclusive = ['macosx']
+
+with Files('subd/**.py'):
+    SCHEDULES.inclusive += ['py-lint']
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/schedules/subd/moz.build
@@ -0,0 +1,2 @@
+with Files('yaml.py'):
+    SCHEDULES.inclusive += ['yaml-lint']
--- a/python/mozbuild/mozbuild/test/frontend/test_reader.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py
@@ -475,11 +475,32 @@ class TestBuildReader(unittest.TestCase)
                              expected_flavors[path])
 
     def test_invalid_flavor(self):
         reader = self.reader('invalid-files-flavor')
 
         with self.assertRaises(BuildReaderError):
             reader.files_info(['foo.js'])
 
+    def test_schedules(self):
+        reader = self.reader('schedules')
+        info = reader.files_info(['somefile', 'foo.win', 'foo.osx', 'subd/aa.py', 'subd/yaml.py'])
+        # default: all exclusive, no inclusive
+        self.assertEqual(info['somefile']['SCHEDULES'].inclusive, [])
+        self.assertEqual(info['somefile']['SCHEDULES'].exclusive, ['android', 'linux', 'macosx', 'windows'])
+        # windows-only
+        self.assertEqual(info['foo.win']['SCHEDULES'].inclusive, [])
+        self.assertEqual(info['foo.win']['SCHEDULES'].exclusive, ['windows'])
+        # osx-only
+        self.assertEqual(info['foo.osx']['SCHEDULES'].inclusive, [])
+        self.assertEqual(info['foo.osx']['SCHEDULES'].exclusive, ['macosx'])
+        # top-level moz.build specifies subd/**.py with an inclusive option
+        self.assertEqual(info['subd/aa.py']['SCHEDULES'].inclusive, ['py-lint'])
+        self.assertEqual(info['subd/aa.py']['SCHEDULES'].exclusive, ['android', 'linux', 'macosx', 'windows'])
+        # Files('yaml.py') in subd/moz.build *overrides* Files('subdir/**.py')
+        self.assertEqual(info['subd/yaml.py']['SCHEDULES'].inclusive, ['yaml-lint'])
+        self.assertEqual(info['subd/yaml.py']['SCHEDULES'].exclusive, ['android', 'linux', 'macosx', 'windows'])
+
+        self.assertEqual(info['subd/yaml.py']['SCHEDULES'].components,
+                ['android', 'linux', 'macosx', 'windows', 'yaml-lint'])
 
 if __name__ == '__main__':
     main()
--- a/python/mozbuild/mozbuild/util.py
+++ b/python/mozbuild/mozbuild/util.py
@@ -480,16 +480,35 @@ class StrictOrderingOnAppendListMixin(ob
 class StrictOrderingOnAppendList(ListMixin, StrictOrderingOnAppendListMixin,
         list):
     """A list specialized for moz.build environments.
 
     We overload the assignment and append operations to require that incoming
     elements be ordered. This enforces cleaner style in moz.build files.
     """
 
+class ImmutableStrictOrderingOnAppendList(StrictOrderingOnAppendList):
+    """Like StrictOrderingOnAppendList, but not allowing mutations of the value.
+    """
+    def append(self, elt):
+        raise Exception("cannot use append on this type")
+
+    def extend(self, iterable):
+        raise Exception("cannot use extend on this type")
+
+    def __setslice__(self, i, j, iterable):
+        raise Exception("cannot assign to slices on this type")
+
+    def __setitem__(self, i, elt):
+        raise Exception("cannot assign to indexes on this type")
+
+    def __iadd__(self, other):
+        raise Exception("cannot use += on this type")
+
+
 class ListWithActionMixin(object):
     """Mixin to create lists with pre-processing. See ListWithAction."""
     def __init__(self, iterable=None, action=None):
         if iterable is None:
             iterable = []
         if not callable(action):
             raise ValueError('A callabe action is required to construct '
                              'a ListWithAction')