Bug 1286075: add support for optimizing based on files changed in the push; r=gps
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 12 Sep 2016 18:40:12 +0000
changeset 313783 315bae6d1488fa36f2f90501218159241866a686
parent 313782 04bd4adb5ed3ca3e2571e8c15dd31ff7a1af3b8b
child 313784 40e59df093a7332f90a34de57a1d08766fc9a93c
push id32255
push userryanvm@gmail.com
push dateWed, 14 Sep 2016 00:47:02 +0000
treeherderautoland@d8f95b350aa2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1286075
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1286075: add support for optimizing based on files changed in the push; r=gps MozReview-Commit-ID: 5di7TuL9X2P
taskcluster/taskgraph/files_changed.py
taskcluster/taskgraph/generator.py
taskcluster/taskgraph/optimize.py
taskcluster/taskgraph/task/base.py
taskcluster/taskgraph/task/docker_image.py
taskcluster/taskgraph/task/legacy.py
taskcluster/taskgraph/task/nightly_fennec.py
taskcluster/taskgraph/task/signing.py
taskcluster/taskgraph/task/transform.py
taskcluster/taskgraph/test/automationrelevance.json
taskcluster/taskgraph/test/test_files_changed.py
taskcluster/taskgraph/test/test_generator.py
taskcluster/taskgraph/test/test_optimize.py
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/files_changed.py
@@ -0,0 +1,65 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Support for optimizing tasks based on the set of files that have changed.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import requests
+from redo import retry
+from mozpack.path import match as mozpackmatch
+
+logger = logging.getLogger(__name__)
+_cache = {}
+
+
+def get_changed_files(repository, revision):
+    """
+    Get the set of files changed in the push headed by the given revision.
+    Responses are cached, so multiple calls with the same arguments are OK.
+    """
+    key = repository, revision
+    if key not in _cache:
+        url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision)
+        logger.debug("Querying version control for metadata: %s", url)
+
+        def get_automationrelevance():
+            response = requests.get(url, timeout=5)
+            return response.json()
+        contents = retry(get_automationrelevance, attempts=2, sleeptime=10)
+
+        logger.debug('{} commits influencing task scheduling:'
+                     .format(len(contents['changesets'])))
+        changed_files = set()
+        for c in contents['changesets']:
+            logger.debug(" {cset} {desc}".format(
+                cset=c['node'][0:12],
+                desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
+            changed_files |= set(c['files'])
+
+        _cache[key] = changed_files
+    return _cache[key]
+
+
+def check(params, file_patterns):
+    """Determine whether any of the files changed in the indicated push to
+    https://hg.mozilla.org match any of the given file patterns."""
+    repository = params.get('head_repository')
+    revision = params.get('head_rev')
+    if not repository or not revision:
+        logger.warning("Missing `head_repository` or `head_rev` parameters; "
+                       "assuming all files have changed")
+        return True
+
+    changed_files = get_changed_files(repository, revision)
+
+    for pattern in file_patterns:
+        for path in changed_files:
+            if mozpackmatch(path, pattern):
+                return True
+
+    return False
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -198,16 +198,17 @@ class TaskGraphGenerator(object):
             target_graph)
         yield 'target_task_graph', target_task_graph
 
         logger.info("Generating optimized task graph")
         do_not_optimize = set()
         if not self.parameters.get('optimize_target_tasks', True):
             do_not_optimize = target_task_set.graph.nodes
         optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
+                                                                    self.parameters,
                                                                     do_not_optimize)
         yield 'label_to_taskid', label_to_taskid
         yield 'optimized_task_graph', optimized_task_graph
 
     def _run_until(self, name):
         while name not in self._run_results:
             try:
                 k, v = self._run.next()
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -9,30 +9,31 @@ import re
 from .graph import Graph
 from .taskgraph import TaskGraph
 from slugid import nice as slugid
 
 logger = logging.getLogger(__name__)
 TASK_REFERENCE_PATTERN = re.compile('<([^>]+)>')
 
 
-def optimize_task_graph(target_task_graph, do_not_optimize, existing_tasks=None):
+def optimize_task_graph(target_task_graph, params, do_not_optimize, existing_tasks=None):
     """
     Perform task optimization, without optimizing tasks named in
     do_not_optimize.
     """
     named_links_dict = target_task_graph.graph.named_links_dict()
     label_to_taskid = {}
 
     # This proceeds in two phases.  First, mark all optimized tasks (those
     # which will be removed from the graph) as such, including a replacement
     # taskId where applicable.  Second, generate a new task graph containing
     # only the non-optimized tasks, with all task labels resolved to taskIds
     # and with task['dependencies'] populated.
     annotate_task_graph(target_task_graph=target_task_graph,
+                        params=params,
                         do_not_optimize=do_not_optimize,
                         named_links_dict=named_links_dict,
                         label_to_taskid=label_to_taskid,
                         existing_tasks=existing_tasks)
     return get_subgraph(target_task_graph, named_links_dict, label_to_taskid), label_to_taskid
 
 
 def resolve_task_references(label, task_def, taskid_for_edge_name):
@@ -54,17 +55,17 @@ def resolve_task_references(label, task_
                 return TASK_REFERENCE_PATTERN.sub(repl, val['task-reference'])
             else:
                 return {k: recurse(v) for k, v in val.iteritems()}
         else:
             return val
     return recurse(task_def)
 
 
-def annotate_task_graph(target_task_graph, do_not_optimize,
+def annotate_task_graph(target_task_graph, params, do_not_optimize,
                         named_links_dict, label_to_taskid, existing_tasks):
     """
     Annotate each task in the graph with .optimized (boolean) and .task_id
     (possibly None), following the rules for optimization and calling the task
     kinds' `optimize_task` method.
 
     As a side effect, label_to_taskid is updated with labels for all optimized
     tasks that are replaced with existing tasks.
@@ -89,17 +90,17 @@ def annotate_task_graph(target_task_grap
         if label in do_not_optimize:
             optimized = False
         # Let's check whether this task has been created before
         elif existing_tasks is not None and label in existing_tasks:
             optimized = True
             replacement_task_id = existing_tasks[label]
         # otherwise, examine the task itself (which may be an expensive operation)
         else:
-            optimized, replacement_task_id = task.optimize()
+            optimized, replacement_task_id = task.optimize(params)
 
         task.optimized = optimized
         task.task_id = replacement_task_id
         if replacement_task_id:
             label_to_taskid[label] = replacement_task_id
 
         if optimized:
             if replacement_task_id:
--- a/taskcluster/taskgraph/task/base.py
+++ b/taskcluster/taskgraph/task/base.py
@@ -74,17 +74,17 @@ class Task(object):
         """
         Get the set of task labels this task depends on, by querying the full
         task set, given as `taskgraph`.
 
         Returns a list of (task_label, dependency_name) pairs describing the
         dependencies.
         """
 
-    def optimize(self):
+    def optimize(self, params):
         """
         Determine whether this task can be optimized, and if it can, what taskId
         it should be replaced with.
 
         The return value is a tuple `(optimized, taskId)`.  If `optimized` is
         true, then the task will be optimized (in other words, not included in
         the task graph).  If the second argument is a taskid, then any
         dependencies on this task will isntead depend on that taskId.  It is an
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -113,17 +113,17 @@ class DockerImageTask(base.Task):
                              task=image_task['task'], attributes=attributes,
                              index_paths=index_paths))
 
         return tasks
 
     def get_dependencies(self, taskgraph):
         return []
 
-    def optimize(self):
+    def optimize(self, params):
         for index_path in self.index_paths:
             try:
                 url = INDEX_URL.format(index_path)
                 existing_task = json.load(urllib2.urlopen(url))
                 # Only return the task ID if the artifact exists for the indexed
                 # task.  Otherwise, continue on looking at each of the branches.  Method
                 # continues trying other branches in case mozilla-central has an expired
                 # artifact, but 'project' might not. Only return no task ID if all
--- a/taskcluster/taskgraph/task/legacy.py
+++ b/taskcluster/taskgraph/task/legacy.py
@@ -613,17 +613,17 @@ class LegacyTask(base.Task):
 
         # add a dependency on an image task, if needed
         if 'docker-image' in self.task_dict:
             deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
                          'docker-image'))
 
         return deps
 
-    def optimize(self):
+    def optimize(self, params):
         # no optimization for the moment
         return False, None
 
     @classmethod
     def from_json(cls, task_dict):
         legacy_task = cls(kind='legacy',
                           label=task_dict['label'],
                           attributes=task_dict['attributes'],
--- a/taskcluster/taskgraph/task/nightly_fennec.py
+++ b/taskcluster/taskgraph/task/nightly_fennec.py
@@ -104,10 +104,10 @@ class NightlyFennecTask(base.Task):
 
         # add a dependency on an image task, if needed
         if 'docker-image' in self.task_dict:
             deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
                          'docker-image'))
 
         return deps
 
-    def optimize(self):
+    def optimize(self, params):
         return False, None
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@@ -46,10 +46,10 @@ class SigningTask(base.Task):
             tasks.append(cls(kind, 'signing-nightly-fennec', task=task['task'],
                              attributes=attributes))
 
         return tasks
 
     def get_dependencies(self, taskgraph):
         return [('build-nightly-fennec', 'build-nightly-fennec')]
 
-    def optimize(self):
+    def optimize(self, params):
         return False, None
--- a/taskcluster/taskgraph/task/transform.py
+++ b/taskcluster/taskgraph/task/transform.py
@@ -2,29 +2,30 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
 
 from . import base
+from .. import files_changed
 from ..util.python_path import find_object
 from ..util.yaml import load_yaml
 from ..transforms.base import TransformSequence, TransformConfig
 
 logger = logging.getLogger(__name__)
 
 
 class TransformTask(base.Task):
     """
     Tasks of this class are generated by applying transformations to a sequence
     of input entities.  By default, it gets those inputs from YAML data in the
-    kind directory, but subclasses may override `get_inputs` to produce them
-    in some other way.
+    kind directory, but subclasses may override `get_inputs` to produce them in
+    some other way.
     """
 
     @classmethod
     def get_inputs(cls, kind, path, config, params, loaded_tasks):
         """
         Get the input elements that will be transformed into tasks.  The
         elements themselves are free-form, and become the input to the first
         transform.
@@ -58,16 +59,24 @@ class TransformTask(base.Task):
 
         # perform the transformations
         trans_config = TransformConfig(kind, path, config, params)
         tasks = [cls(kind, t) for t in transforms(trans_config, inputs)]
         return tasks
 
     def __init__(self, kind, task):
         self.dependencies = task['dependencies']
+        self.when = task['when']
         super(TransformTask, self).__init__(kind, task['label'],
                                             task['attributes'], task['task'])
 
     def get_dependencies(self, taskgraph):
         return [(label, name) for name, label in self.dependencies.items()]
 
-    def optimize(self):
+    def optimize(self, params):
+        if 'files-changed' in self.when:
+            changed = files_changed.check(
+                params, self.when['files-changed'])
+            if not changed:
+                logger.debug('no files found matching a pattern in `when.files-changed` for '
+                             + self.label)
+                return True, None
         return False, None
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/automationrelevance.json
@@ -0,0 +1,425 @@
+{
+    "changesets": [
+        {
+            "author": "James Long <longster@gmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1300866",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866"
+                }
+            ],
+            "date": [
+                1473196655.0,
+                14400
+            ],
+            "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/debugger/new/index.html"
+            ],
+            "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "parents": [
+                "37c9349b4e8167a61b08b7e119c21ea177b98942"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312890,
+            "reviewers": [
+                {
+                    "name": "jlast",
+                    "revset": "reviewer(jlast)"
+                },
+                {
+                    "name": "bgrins",
+                    "revset": "reviewer(bgrins)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Wes Kocher <wkocher@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [],
+            "date": [
+                1473208638.0,
+                25200
+            ],
+            "desc": "Merge m-c to fx-team, a=merge",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "taskcluster/scripts/builder/build-l10n.sh"
+            ],
+            "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+            "parents": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312891,
+            "reviewers": [
+                {
+                    "name": "merge",
+                    "revset": "reviewer(merge)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Towkir Ahmed <towkir17@gmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1296648",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648"
+                }
+            ],
+            "date": [
+                1472957580.0,
+                14400
+            ],
+            "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/themes/rules.css"
+            ],
+            "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+            "parents": [
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312892,
+            "reviewers": [
+                {
+                    "name": "ntim",
+                    "revset": "reviewer(ntim)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Oriol <oriol-bugzilla@hotmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1300336",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336"
+                }
+            ],
+            "date": [
+                1472921160.0,
+                14400
+            ],
+            "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/webconsole/test/browser_webconsole_output_06.js",
+                "devtools/server/actors/object.js"
+            ],
+            "node": "99c542fa43a72ee863c813b5624048d1b443549b",
+            "parents": [
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312893,
+            "reviewers": [
+                {
+                    "name": "fitzgen",
+                    "revset": "reviewer(fitzgen)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Ruturaj Vartak <ruturaj@gmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1295010",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
+                }
+            ],
+            "date": [
+                1472854020.0,
+                -7200
+            ],
+            "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK",
+            "extra": {
+                "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd",
+                "branch": "default",
+                "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba",
+                "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651"
+            },
+            "files": [
+                "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
+                "devtools/client/inspector/test/head.js",
+                "devtools/server/actors/highlighters/eye-dropper.js"
+            ],
+            "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+            "parents": [
+                "99c542fa43a72ee863c813b5624048d1b443549b"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312894,
+            "reviewers": [
+                {
+                    "name": "pbro",
+                    "revset": "reviewer(pbro)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Matteo Ferretti <mferretti@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1299154",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154"
+                }
+            ],
+            "date": [
+                1472629906.0,
+                -7200
+            ],
+            "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n",
+            "extra": {
+                "branch": "default",
+                "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200"
+            },
+            "files": [
+                "docshell/base/nsDocShell.cpp",
+                "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html"
+            ],
+            "node": "541c9086c0f27fba60beecc9bc94543103895c86",
+            "parents": [
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312895,
+            "reviewers": [
+                {
+                    "name": "mstange",
+                    "revset": "reviewer(mstange)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Patrick Brosset <pbrosset@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1295010",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
+                }
+            ],
+            "date": [
+                1473239449.0,
+                -7200
+            ],
+            "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
+                "devtools/client/inspector/test/head.js"
+            ],
+            "node": "041a925171e431bf51fb50193ab19d156088c89a",
+            "parents": [
+                "541c9086c0f27fba60beecc9bc94543103895c86"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312896,
+            "reviewers": [
+                {
+                    "name": "me",
+                    "revset": "reviewer(me)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [],
+            "date": [
+                1473261233.0,
+                -7200
+            ],
+            "desc": "merge fx-team to mozilla-central a=merge",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [],
+            "node": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "parents": [
+                "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9",
+                "041a925171e431bf51fb50193ab19d156088c89a"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312897,
+            "reviewers": [
+                {
+                    "name": "merge",
+                    "revset": "reviewer(merge)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        }
+    ],
+    "visible": true
+}
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_files_changed.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import os
+import unittest
+
+from .. import files_changed
+
+PARAMS = {
+    'head_repository': 'https://hg.mozilla.org/mozilla-central',
+    'head_rev': 'a14f88a9af7a',
+}
+
+FILES_CHANGED = [
+    'devtools/client/debugger/new/index.html',
+    'devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js',
+    'devtools/client/inspector/test/head.js',
+    'devtools/client/themes/rules.css',
+    'devtools/client/webconsole/test/browser_webconsole_output_06.js',
+    'devtools/server/actors/highlighters/eye-dropper.js',
+    'devtools/server/actors/object.js',
+    'docshell/base/nsDocShell.cpp',
+    'dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html',
+    'taskcluster/scripts/builder/build-l10n.sh',
+]
+
+
+class FakeResponse:
+
+    def json(self):
+        with open(os.path.join(os.path.dirname(__file__), 'automationrelevance.json')) as f:
+            return json.load(f)
+
+
+class TestGetChangedFiles(unittest.TestCase):
+
+    def setUp(self):
+        files_changed._cache.clear()
+        self.old_get = files_changed.requests.get
+
+        def fake_get(url, **kwargs):
+            return FakeResponse()
+        files_changed.requests.get = fake_get
+
+    def tearDown(self):
+        files_changed.requests.get = self.old_get
+
+    def test_get_changed_files(self):
+        """Get_changed_files correctly gets the list of changed files in a push.
+        This tests against the production hg.mozilla.org so that it will detect
+        any changes in the format of the returned data."""
+        self.assertEqual(
+            sorted(files_changed.get_changed_files(PARAMS['head_repository'], PARAMS['head_rev'])),
+            FILES_CHANGED)
+
+
+class TestCheck(unittest.TestCase):
+
+    def setUp(self):
+        files_changed._cache[PARAMS['head_repository'], PARAMS['head_rev']] = FILES_CHANGED
+
+    def test_check_no_params(self):
+        self.assertTrue(files_changed.check({}, ["ignored"]))
+
+    def test_check_no_match(self):
+        self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"]))
+
+    def test_check_match(self):
+        self.assertTrue(files_changed.check(PARAMS, ["devtools/**"]))
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -29,17 +29,17 @@ class FakeTask(base.Task):
 
     def get_dependencies(self, full_task_set):
         i = self.i
         if i > 0:
             return [('{}-t-{}'.format(self.kind, i - 1), 'prev')]
         else:
             return []
 
-    def optimize(self):
+    def optimize(self, params):
         return False, None
 
 
 class FakeKind(Kind):
 
     def _get_impl_class(self):
         return FakeTask
 
--- a/taskcluster/taskgraph/test/test_optimize.py
+++ b/taskcluster/taskgraph/test/test_optimize.py
@@ -80,85 +80,85 @@ class TestOptimize(unittest.TestCase):
             return 'SLUGID' if task_id and len(task_id) == 22 else task_id
         got_annotations = {
             t.label: (t.optimized, repl(t.task_id)) for t in graph.tasks.itervalues()
         }
         self.assertEqual(got_annotations, annotations)
 
     def test_annotate_task_graph_no_optimize(self):
         "annotating marks everything as un-optimized if the kind returns that"
-        OptimizingTask.optimize = lambda self: (False, None)
+        OptimizingTask.optimize = lambda self, params: (False, None)
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             self.make_task('task3'),
             ('task2', 'task1', 'build'),
             ('task2', 'task3', 'image'),
         )
-        annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
+        annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
         self.assert_annotations(
             graph,
             task1=(False, None),
             task2=(False, None),
             task3=(False, None)
         )
 
     def test_annotate_task_graph_taskid_without_optimize(self):
         "raises exception if kind returns a taskid without optimizing"
-        OptimizingTask.optimize = lambda self: (False, 'some-taskid')
+        OptimizingTask.optimize = lambda self, params: (False, 'some-taskid')
         graph = self.make_graph(self.make_task('task1'))
         self.assertRaises(
             Exception,
-            lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
+            lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
         )
 
     def test_annotate_task_graph_optimize_away_dependency(self):
         "raises exception if kind optimizes away a task on which another depends"
         OptimizingTask.optimize = \
-            lambda self: (True, None) if self.label == 'task1' else (False, None)
+            lambda self, params: (True, None) if self.label == 'task1' else (False, None)
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             ('task2', 'task1', 'build'),
         )
         self.assertRaises(
             Exception,
-            lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
+            lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
         )
 
     def test_annotate_task_graph_do_not_optimize(self):
         "annotating marks everything as un-optimized if in do_not_optimize"
-        OptimizingTask.optimize = lambda self: (True, 'taskid')
+        OptimizingTask.optimize = lambda self, params: (True, 'taskid')
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             ('task2', 'task1', 'build'),
         )
         label_to_taskid = {}
-        annotate_task_graph(graph, {'task1', 'task2'},
+        annotate_task_graph(graph, {}, {'task1', 'task2'},
                             graph.graph.named_links_dict(), label_to_taskid, None)
         self.assert_annotations(
             graph,
             task1=(False, None),
             task2=(False, None)
         )
         self.assertEqual
 
     def test_annotate_task_graph_nos_do_not_propagate(self):
         "a task with a non-optimized dependency can be optimized"
         OptimizingTask.optimize = \
-            lambda self: (False, None) if self.label == 'task1' else (True, 'taskid')
+            lambda self, params: (False, None) if self.label == 'task1' else (True, 'taskid')
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             self.make_task('task3'),
             ('task2', 'task1', 'build'),
             ('task2', 'task3', 'image'),
         )
-        annotate_task_graph(graph, set(),
+        annotate_task_graph(graph, {}, set(),
                             graph.graph.named_links_dict(), {}, None)
         self.assert_annotations(
             graph,
             task1=(False, None),
             task2=(True, 'taskid'),
             task3=(True, 'taskid')
         )
 
@@ -237,20 +237,20 @@ class TestOptimize(unittest.TestCase):
         self.assertEqual(sub.tasks[task2].task_id, task2)
         self.assertEqual(sorted(sub.tasks[task2].task['dependencies']), sorted([task3, 'dep1']))
         self.assertEqual(sub.tasks[task2].task['payload'], 'http://dep1/' + task3)
         self.assertEqual(sub.tasks[task3].task_id, task3)
 
     def test_optimize(self):
         "optimize_task_graph annotates and extracts the subgraph from a simple graph"
         OptimizingTask.optimize = \
-            lambda self: (True, 'dep1') if self.label == 'task1' else (False, None)
+            lambda self, params: (True, 'dep1') if self.label == 'task1' else (False, None)
         input = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             self.make_task('task3'),
             ('task2', 'task1', 'build'),
             ('task2', 'task3', 'image'),
         )
-        opt, label_to_taskid = optimize_task_graph(input, set())
+        opt, label_to_taskid = optimize_task_graph(input, {}, set())
         self.assertEqual(opt.graph, graph.Graph(
             {label_to_taskid['task2'], label_to_taskid['task3']},
             {(label_to_taskid['task2'], label_to_taskid['task3'], 'image')}))
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -49,16 +49,17 @@ job_description_schema = Schema({
     Optional('scopes'): task_description_schema['scopes'],
     Optional('extra'): task_description_schema['extra'],
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
     Optional('coalesce-name'): task_description_schema['coalesce-name'],
     Optional('worker-type'): task_description_schema['worker-type'],
     Required('worker'): task_description_schema['worker'],
+    Optional('when'): task_description_schema['when'],
 
     # A description of how to run this job.
     'run': {
         # The key to a job implementation in a peer module to this one
         'using': basestring,
 
         # Any remaining content is verified against that job implementation's
         # own schema.
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -207,16 +207,26 @@ task_description_schema = Schema({
             Optional('repository'): basestring,
             Optional('project'): basestring,
         },
         'properties': {
             'product': basestring,
             Extra: basestring,  # additional properties are allowed
         },
     }),
+
+    # The "when" section contains descriptions of the circumstances
+    # under which this task can be "optimized", that is, left out of the
+    # task graph because it is unnecessary.
+    Optional('when'): Any({
+        # This task only needs to be run if a file matching one of the given
+        # patterns has changed in the push.  The patterns use the mozpack
+        # match function (python/mozbuild/mozpack/path.py).
+        Optional('files-changed'): [basestring],
+    }),
 })
 
 GROUP_NAMES = {
     'tc': 'Executed by TaskCluster',
     'tc-e10s': 'Executed by TaskCluster with e10s',
     'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster',
     'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s',
     'tc-Fxfn-r': 'Firefox functional tests (remote) executed by TaskCluster',