bug 1344321 - add nightly test support. r=dustin a=release
authorAki Sasaki <asasaki@mozilla.com>
Mon, 13 Mar 2017 17:09:14 -0700
changeset 379058 d5d5a6044fc54d03f932cd21d2ea7250b51ff486
parent 379057 fa57d7bb08727b72deca53dcd91c2041443e82e7
child 379059 5e13ff0344cc8b3a67a94fd05f7a973164ef2977
push id1419
push userjlund@mozilla.com
push dateMon, 10 Apr 2017 20:44:07 +0000
treeherdermozilla-release@5e6801b73ef6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin, release
bugs1344321
milestone53.0
bug 1344321 - add nightly test support. r=dustin a=release this patch: - adds linux{32,64}-nightly/opt test platforms that mirror the non-nightly test platforms. - adds an `include_nightly` per-project parameter; this is refered to in the default `target_task_method`. It's still possible to launch custom `target_task_method`s to trigger nightlies against, say,try. - adds a `filter_for_project` method in `target_tasks.py` that allows for `include_nightly` and `run_on_projects` filtering in the various `target_task_method`s. - adds nightly filtering into the `TryOptionSyntax` object. By default, this will be off. To trigger nightly tests on try, either submit a new decision task with a different `target_task_method` (e.g. `nightly_fennec`) or flip the `include_nightly` flag to True. - adds the `nightly` attribute to tests if their builds have that attribute. MozReview-Commit-ID: DttIZH0BHS2
taskcluster/docs/parameters.rst
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/task/test.py
taskcluster/taskgraph/try_option_syntax.py
--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@@ -95,8 +95,11 @@ syntax or reading a project-specific con
     global.
 
 ``target_tasks_method``
     The method to use to determine the target task set.  This is the suffix of
     one of the functions in ``tascluster/taskgraph/target_tasks.py``.
 
 ``optimize_target_tasks``
    If true, then target tasks are eligible for optimization.
+
+``include_nightly``
+   If true, then nightly tasks are eligible for optimization.
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -33,46 +33,53 @@ GECKO = os.path.realpath(os.path.join(__
 # See `taskcluster/docs/parameters.rst` for information on parameters.
 PER_PROJECT_PARAMETERS = {
     'try': {
         'target_tasks_method': 'try_option_syntax',
         # Always perform optimization.  This makes it difficult to use try
         # pushes to run a task that would otherwise be optimized, but is a
         # compromise to avoid essentially disabling optimization in try.
         'optimize_target_tasks': True,
+        'include_nightly': False,
     },
 
     'ash': {
         'target_tasks_method': 'ash_tasks',
         'optimize_target_tasks': True,
+        'include_nightly': False,
     },
 
     'cedar': {
         'target_tasks_method': 'cedar_tasks',
         'optimize_target_tasks': True,
+        'include_nightly': False,
     },
 
     'graphics': {
         'target_tasks_method': 'graphics_tasks',
         'optimize_target_tasks': True,
+        'include_nightly': False,
     },
 
     'mozilla-beta': {
         'target_tasks_method': 'mozilla_beta_tasks',
         'optimize_target_tasks': True,
+        'include_nightly': True,
     },
     'mozilla-release': {
         'target_tasks_method': 'mozilla_release_tasks',
         'optimize_target_tasks': True,
+        'include_nightly': True,
     },
 
     # the default parameters are used for projects that do not match above.
     'default': {
         'target_tasks_method': 'default',
         'optimize_target_tasks': True,
+        'include_nightly': False,
     }
 }
 
 
 def taskgraph_decision(options):
     """
     Run the decision task.  This function implements `mach taskgraph decision`,
     and is responsible for
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -18,16 +18,24 @@ def _target_task(name):
     return wrap
 
 
 def get_method(method):
     """Get a target_task_method to pass to a TaskGraphGenerator."""
     return _target_task_methods[method]
 
 
+def filter_for_project(task, parameters):
+    """Filter tasks by project.  Optionally enable nightlies."""
+    if task.attributes.get('nightly') and not parameters.get('include_nightly'):
+        return False
+    run_on_projects = set(task.attributes.get('run_on_projects', []))
+    return match_run_on_projects(parameters['project'], run_on_projects)
+
+
 @_target_task('try_option_syntax')
 def target_tasks_try_option_syntax(full_task_graph, parameters):
     """Generate a list of target tasks based on try syntax in
     parameters['message'] and, for context, the full task graph."""
     options = try_option_syntax.TryOptionSyntax(parameters['message'], full_task_graph)
     target_tasks_labels = [t.label for t in full_task_graph.tasks.itervalues()
                            if options.task_matches(t.attributes)]
 
@@ -51,20 +59,19 @@ def target_tasks_try_option_syntax(full_
 
     return target_tasks_labels
 
 
 @_target_task('default')
 def target_tasks_default(full_task_graph, parameters):
     """Target the tasks which have indicated they should be run on this project
     via the `run_on_projects` attributes."""
-    def filter(task):
-        run_on_projects = set(task.attributes.get('run_on_projects', []))
-        return match_run_on_projects(parameters['project'], run_on_projects)
-    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+    return [l for l, t in full_task_graph.tasks.iteritems()
+            if filter_for_project(t, parameters)]
 
 
 @_target_task('ash_tasks')
 def target_tasks_ash(full_task_graph, parameters):
     """Target tasks that only run on the ash branch."""
     def filter(task):
         platform = task.attributes.get('build_platform')
         # only select platforms
@@ -139,32 +146,35 @@ def target_tasks_nightly_linux(full_task
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('mozilla_beta_tasks')
 def target_tasks_mozilla_beta(full_task_graph, parameters):
     """Select the set of tasks required for a promotable beta or release build
     of linux, plus android CI. The candidates build process involves a pipeline
     of builds and signing, but does not include beetmover or balrog jobs."""
+
     def filter(task):
         platform = task.attributes.get('build_platform')
         if platform in ('android-api-15', 'android-x86', 'linux64-asan',
                         'linux64-pgo', 'linux64-add-on-devel'):
             return True
         if platform in ('linux64', 'linux'):
             if task.attributes['build_type'] == 'debug':
                 return True
         if platform in ('linux64-nightly', 'linux-nightly'):
             if task.kind in ["test"]:
                 return True
+            # skip l10n, beetmover, balrog
             if task.kind not in [
                 'balrog', 'beetmover', 'beetmover-checksums', 'beetmover-l10n',
                 'checksums-signing', 'nightly-l10n', 'nightly-l10n-signing'
             ]:
-                return task.attributes.get('nightly', False)
+                return True
+
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('mozilla_release_tasks')
 def target_tasks_mozilla_release(full_task_graph, parameters):
     """Select the set of tasks required for a promotable beta or release build
     of linux, plus android CI. The candidates build process involves a pipeline
     of builds and signing, but does not include beetmover or balrog jobs."""
--- a/taskcluster/taskgraph/task/test.py
+++ b/taskcluster/taskgraph/task/test.py
@@ -43,57 +43,59 @@ class TestTask(transform.TransformTask):
         # generate all tests for all test platforms
         for test_platform_name, test_platform in test_platforms.iteritems():
             for test_name in test_platform['test-names']:
                 test = copy.deepcopy(test_descriptions[test_name])
                 test['build-platform'] = test_platform['build-platform']
                 test['test-platform'] = test_platform_name
                 test['build-label'] = test_platform['build-label']
                 test['test-name'] = test_name
+                if test_platform['nightly']:
+                    test.setdefault('attributes', {})['nightly'] = True
 
                 logger.debug("Generating tasks for test {} on platform {}".format(
                     test_name, test['test-platform']))
                 yield test
 
     @classmethod
     def get_builds_by_platform(cls, dep_kind, loaded_tasks):
         """Find the build tasks on which tests will depend, keyed by
-        platform/type.  Returns a dictionary mapping build platform to task
-        label."""
+        platform/type.  Returns a dictionary mapping build platform to task."""
         builds_by_platform = {}
         for task in loaded_tasks:
             if task.kind != dep_kind:
                 continue
 
             build_platform = task.attributes.get('build_platform')
             build_type = task.attributes.get('build_type')
             if not build_platform or not build_type:
                 continue
             platform = "{}/{}".format(build_platform, build_type)
             if platform in builds_by_platform:
                 raise Exception("multiple build jobs for " + platform)
-            builds_by_platform[platform] = task.label
+            builds_by_platform[platform] = task
         return builds_by_platform
 
     @classmethod
     def get_test_platforms(cls, test_platforms_cfg, builds_by_platform):
         """Get the test platforms for which test tasks should be generated,
         based on the available build platforms.  Returns a dictionary mapping
         test platform to {test-set, build-platform, build-label}."""
         test_platforms = {}
         for test_platform, cfg in test_platforms_cfg.iteritems():
             build_platform = cfg['build-platform']
             if build_platform not in builds_by_platform:
                 logger.warning(
                     "No build task with platform {}; ignoring test platform {}".format(
                         build_platform, test_platform))
                 continue
             test_platforms[test_platform] = {
+                'nightly': builds_by_platform[build_platform].attributes.get('nightly', False),
                 'build-platform': build_platform,
-                'build-label': builds_by_platform[build_platform],
+                'build-label': builds_by_platform[build_platform].label,
             }
             test_platforms[test_platform].update(cfg)
         return test_platforms
 
     @classmethod
     def expand_tests(cls, test_sets_cfg, test_platforms):
         """Expand the test sets in `test_platforms` out to sets of test names.
         Returns a dictionary like `get_test_platforms`, with an additional
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -51,16 +51,17 @@ def alias_prefix(prefix):
 def alias_contains(infix):
     return lambda name: infix in name
 
 
 def alias_matches(pattern):
     pattern = re.compile(pattern)
     return lambda name: pattern.match(name)
 
+
 UNITTEST_ALIASES = {
     # Aliases specify shorthands that can be used in try syntax.  The shorthand
     # is the dictionary key, with the value representing a pattern for matching
     # unittest_try_names.
     #
     # Note that alias expansion is performed in the absence of any chunk
     # prefixes.  For example, the first example above would replace "foo-7"
     # with "foobar-7".  Note that a few aliases allowed chunks to be specified
@@ -230,29 +231,31 @@ class TryOptionSyntax(object):
         parser.add_argument('-t', '--talos', nargs='?', dest='talos', const='all', default='all')
         parser.add_argument('-i', '--interactive',
                             dest='interactive', action='store_true', default=False)
         parser.add_argument('-e', '--all-emails',
                             dest='notifications', action='store_const', const='all')
         parser.add_argument('-f', '--failure-emails',
                             dest='notifications', action='store_const', const='failure')
         parser.add_argument('-j', '--job', dest='jobs', action='append')
+        parser.add_argument('--include-nightly', dest='include_nightly', action='store_true')
         # In order to run test jobs multiple times
         parser.add_argument('--rebuild', dest='trigger_tests', type=int, default=1)
         args, _ = parser.parse_known_args(parts[try_idx:])
 
         self.jobs = self.parse_jobs(args.jobs)
         self.build_types = self.parse_build_types(args.build_types)
         self.platforms = self.parse_platforms(args.platforms)
         self.unittests = self.parse_test_option(
             "unittest_try_name", args.unittests, full_task_graph)
         self.talos = self.parse_test_option("talos_try_name", args.talos, full_task_graph)
         self.trigger_tests = args.trigger_tests
         self.interactive = args.interactive
         self.notifications = args.notifications
+        self.include_nightly = args.include_nightly
 
     def parse_jobs(self, jobs_arg):
         if not jobs_arg or jobs_arg == ['all']:
             return None
         expanded = []
         for job in jobs_arg:
             expanded.extend(j.strip() for j in job.split(','))
         return expanded
@@ -493,16 +496,18 @@ class TryOptionSyntax(object):
             result += char
 
         return result
 
     def task_matches(self, attributes):
         attr = attributes.get
 
         def check_run_on_projects():
+            if attr('nightly') and not self.include_nightly:
+                return False
             return set(['try', 'all']) & set(attr('run_on_projects', []))
 
         def match_test(try_spec, attr_name):
             if attr('build_type') not in self.build_types:
                 return False
             if self.platforms is not None:
                 if attr('build_platform') not in self.platforms:
                     return False