Bug 1334167: use run-on-projects to parallel task graph generation; r=Callek
authorDustin J. Mitchell <dustin@mozilla.com>
Tue, 31 Jan 2017 19:49:18 +0000
changeset 332267 ae6c320b791b541c07f40e71b568a02c73d80752
parent 332266 f3ebbf9445ab3ed08987ab8932fd584b4a1f2914
child 332268 3cb163fd021d30f2e6e4a9eab8f3d1c70dfd5c2c
push id86501
push userkwierso@gmail.com
push dateFri, 03 Feb 2017 00:45:15 +0000
treeherdermozilla-inbound@f3793c0f52fa [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek
bugs1334167
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1334167: use run-on-projects to parallel task graph generation; r=Callek MozReview-Commit-ID: EQMuh4hN9Ya
.cron.yml
taskcluster/docs/cron.rst
taskcluster/taskgraph/cron/__init__.py
taskcluster/taskgraph/cron/schema.py
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/test/test_util_attributes.py
taskcluster/taskgraph/util/attributes.py
--- a/.cron.yml
+++ b/.cron.yml
@@ -4,26 +4,26 @@
 
 jobs:
     - name: nightly-desktop
       job:
           type: decision-task
           treeherder-symbol: Nd
           triggered-by: nightly
           target-tasks-method: nightly_linux
-      projects:
+      run-on-projects:
           - mozilla-central
           - date
       when:
           - {hour: 16, minute: 0}
 
     - name: nightly-android
       job:
           type: decision-task
           treeherder-symbol: Na
           triggered-by: nightly
           target-tasks-method: nightly_fennec
-      projects:
+      run-on-projects:
           - mozilla-central
           - date
       when:
           - {hour: 16, minute: 0}
 
--- a/taskcluster/docs/cron.rst
+++ b/taskcluster/docs/cron.rst
@@ -1,14 +1,24 @@
 Periodic Taskgraphs
 ===================
 
 The cron functionality allows in-tree scheduling of task graphs that run
 periodically, instead of on a push.
 
+Cron.yml
+--------
+
+In the root of the Gecko directory, you will find `.cron.yml`.  This defines
+the periodic tasks ("cron jobs") run for Gecko.  Each specifies a name, what to
+do, and some parameters to determine when the cron job should occur.
+
+See ``taskcluster/taskgraph/cron/schema.py`` for details on the format and
+meaning of this file.
+
 How It Works
 ------------
 
 The `TaskCluster Hooks Service <https://tools.taskcluster.net/hooks>`_ has a
 hook configured for each repository supporting periodic task graphs.  The hook
 runs every 15 minutes, and the resulting task is referred to as a "cron task".
 That cron task runs `./mach taskgraph cron` in a checkout of the Gecko source
 tree.
--- a/taskcluster/taskgraph/cron/__init__.py
+++ b/taskcluster/taskgraph/cron/__init__.py
@@ -16,16 +16,17 @@ import requests
 import yaml
 
 from . import decision, schema
 from .util import (
     match_utc,
     calculate_head_rev
 )
 from ..create import create_task
+from taskgraph.util.attributes import match_run_on_projects
 
 # Functions to handle each `job.type` in `.cron.yml`.  These are called with
 # the contents of the `job` property from `.cron.yml` and should return a
 # sequence of (taskId, task) tuples which will subsequently be fed to
 # createTask.
 JOB_TYPES = {
     'decision-task': decision.run_decision_task,
 }
@@ -45,19 +46,19 @@ def get_session():
 def load_jobs():
     with open(os.path.join(GECKO, '.cron.yml'), 'rb') as f:
         cron_yml = yaml.load(f)
     schema.validate(cron_yml)
     return {j['name']: j for j in cron_yml['jobs']}
 
 
 def should_run(job, params):
-    if 'projects' in job:
-        if not any(p == params['project'] for p in job['projects']):
-            return False
+    run_on_projects = job.get('run-on-projects', ['all'])
+    if not match_run_on_projects(params['project'], run_on_projects):
+        return False
     if not any(match_utc(params, hour=sched.get('hour'), minute=sched.get('minute'))
                for sched in job.get('when', [])):
         return False
     return True
 
 
 def run_job(job_name, job, params):
     params['job_name'] = job_name
--- a/taskcluster/taskgraph/cron/schema.py
+++ b/taskcluster/taskgraph/cron/schema.py
@@ -33,18 +33,20 @@ cron_yml_schema = Schema({
 
             # --target-tasks-method './mach taskgraph decision' argument
             'target-tasks-method': basestring,
         }),
 
         # when to run it
 
         # Optional set of projects on which this job should run; if omitted, this will
-        # run on all projects for which cron tasks are set up
-        'projects': [basestring],
+        # run on all projects for which cron tasks are set up.  This works just like the
+        # `run_on_projects` attribute, where strings like "release" and "integration" are
+        # expanded to cover multiple repositories.  (taskcluster/docs/attributes.rst)
+        'run-on-projects': [basestring],
 
         # Array of times at which this task should run.  These *must* be a multiple of
         # 15 minutes, the minimum scheduling interval.
         'when': [{'hour': int, 'minute': All(int, even_15_minutes)}],
     }],
 })
 
 
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -1,28 +1,17 @@
 # -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 from taskgraph import try_option_syntax
-
-INTEGRATION_PROJECTS = set([
-    'mozilla-inbound',
-    'autoland',
-])
-
-RELEASE_PROJECTS = set([
-    'mozilla-central',
-    'mozilla-aurora',
-    'mozilla-beta',
-    'mozilla-release',
-])
+from taskgraph.util.attributes import match_run_on_projects
 
 _target_task_methods = {}
 
 
 def _target_task(name):
     def wrap(func):
         _target_task_methods[name] = func
         return func
@@ -84,26 +73,17 @@ def target_tasks_try_option_syntax(full_
 
 
 @_target_task('default')
 def target_tasks_default(full_task_graph, parameters):
     """Target the tasks which have indicated they should be run on this project
     via the `run_on_projects` attributes."""
     def filter(task):
         run_on_projects = set(task.attributes.get('run_on_projects', []))
-        if 'all' in run_on_projects:
-            return True
-        project = parameters['project']
-        if 'integration' in run_on_projects:
-            if project in INTEGRATION_PROJECTS:
-                return True
-        if 'release' in run_on_projects:
-            if project in RELEASE_PROJECTS:
-                return True
-        return project in run_on_projects
+        return match_run_on_projects(parameters['project'], run_on_projects)
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
 @_target_task('ash_tasks')
 def target_tasks_ash(full_task_graph, parameters):
     """Target tasks that only run on the ash branch."""
     def filter(task):
         platform = task.attributes.get('build_platform')
--- a/taskcluster/taskgraph/test/test_util_attributes.py
+++ b/taskcluster/taskgraph/test/test_util_attributes.py
@@ -1,16 +1,19 @@
 # -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import unittest
-from taskgraph.util.attributes import attrmatch
+from taskgraph.util.attributes import (
+    attrmatch,
+    match_run_on_projects,
+)
 
 
 class Attrmatch(unittest.TestCase):
 
     def test_trivial_match(self):
         """Given no conditions, anything matches"""
         self.assertTrue(attrmatch({}))
 
@@ -38,8 +41,55 @@ class Attrmatch(unittest.TestCase):
         self.assertTrue(attrmatch({'att': 10}, att=even))
         self.assertFalse(attrmatch({'att': 11}, att=even))
 
     def test_all_matches_required(self):
         """If only one attribute does not match, the result is False"""
         self.assertFalse(attrmatch({'a': 1}, a=1, b=2, c=3))
         self.assertFalse(attrmatch({'a': 1, 'b': 2}, a=1, b=2, c=3))
         self.assertTrue(attrmatch({'a': 1, 'b': 2, 'c': 3}, a=1, b=2, c=3))
+
+
+class MatchRunOnProjects(unittest.TestCase):
+
+    def test_empty(self):
+        self.assertFalse(match_run_on_projects('try', []))
+
+    def test_all(self):
+        self.assertTrue(match_run_on_projects('try', ['all']))
+        self.assertTrue(match_run_on_projects('larch', ['all']))
+        self.assertTrue(match_run_on_projects('autoland', ['all']))
+        self.assertTrue(match_run_on_projects('mozilla-inbound', ['all']))
+        self.assertTrue(match_run_on_projects('mozilla-central', ['all']))
+        self.assertTrue(match_run_on_projects('mozilla-aurora', ['all']))
+        self.assertTrue(match_run_on_projects('mozilla-beta', ['all']))
+        self.assertTrue(match_run_on_projects('mozilla-release', ['all']))
+
+    def test_release(self):
+        self.assertFalse(match_run_on_projects('try', ['release']))
+        self.assertFalse(match_run_on_projects('larch', ['release']))
+        self.assertFalse(match_run_on_projects('autoland', ['release']))
+        self.assertFalse(match_run_on_projects('mozilla-inbound', ['release']))
+        self.assertTrue(match_run_on_projects('mozilla-central', ['release']))
+        self.assertTrue(match_run_on_projects('mozilla-aurora', ['release']))
+        self.assertTrue(match_run_on_projects('mozilla-beta', ['release']))
+        self.assertTrue(match_run_on_projects('mozilla-release', ['release']))
+
+    def test_integration(self):
+        self.assertFalse(match_run_on_projects('try', ['integration']))
+        self.assertFalse(match_run_on_projects('larch', ['integration']))
+        self.assertTrue(match_run_on_projects('autoland', ['integration']))
+        self.assertTrue(match_run_on_projects('mozilla-inbound', ['integration']))
+        self.assertFalse(match_run_on_projects('mozilla-central', ['integration']))
+        self.assertFalse(match_run_on_projects('mozilla-aurora', ['integration']))
+        self.assertFalse(match_run_on_projects('mozilla-beta', ['integration']))
+        self.assertFalse(match_run_on_projects('mozilla-integration', ['integration']))
+
+    def test_combo(self):
+        self.assertTrue(match_run_on_projects('try', ['release', 'try', 'date']))
+        self.assertFalse(match_run_on_projects('larch', ['release', 'try', 'date']))
+        self.assertTrue(match_run_on_projects('date', ['release', 'try', 'date']))
+        self.assertFalse(match_run_on_projects('autoland', ['release', 'try', 'date']))
+        self.assertFalse(match_run_on_projects('mozilla-inbound', ['release', 'try', 'date']))
+        self.assertTrue(match_run_on_projects('mozilla-central', ['release', 'try', 'date']))
+        self.assertTrue(match_run_on_projects('mozilla-aurora', ['release', 'try', 'date']))
+        self.assertTrue(match_run_on_projects('mozilla-beta', ['release', 'try', 'date']))
+        self.assertTrue(match_run_on_projects('mozilla-release', ['release', 'try', 'date']))
--- a/taskcluster/taskgraph/util/attributes.py
+++ b/taskcluster/taskgraph/util/attributes.py
@@ -1,12 +1,24 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+INTEGRATION_PROJECTS = set([
+    'mozilla-inbound',
+    'autoland',
+])
+
+RELEASE_PROJECTS = set([
+    'mozilla-central',
+    'mozilla-aurora',
+    'mozilla-beta',
+    'mozilla-release',
+])
+
 
 def attrmatch(attributes, **kwargs):
     """Determine whether the given set of task attributes matches.  The
     conditions are given as keyword arguments, where each keyword names an
     attribute.  The keyword value can be a literal, a set, or a callable.  A
     literal must match the attribute exactly.  Given a set, the attribute value
     must be in the set.  A callable is called with the attribute value.  If an
     attribute is specified as a keyword argument but not present in the
@@ -19,8 +31,23 @@ def attrmatch(attributes, **kwargs):
             if attval not in kwval:
                 return False
         elif callable(kwval):
             if not kwval(attval):
                 return False
         elif kwval != attributes[kwkey]:
             return False
     return True
+
+
+def match_run_on_projects(project, run_on_projects):
+    """Determine whether the given project is included in the `run-on-projects`
+    parameter, applying expansions for things like "integration" mentioned in
+    the attribute documentation."""
+    if 'all' in run_on_projects:
+        return True
+    if 'integration' in run_on_projects:
+        if project in INTEGRATION_PROJECTS:
+            return True
+    if 'release' in run_on_projects:
+        if project in RELEASE_PROJECTS:
+            return True
+    return project in run_on_projects