Bug 1286075: add support for optimizing based on files changed in the push; r=gps
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 12 Sep 2016 18:40:12 +0000
changeset 313705 315bae6d1488fa36f2f90501218159241866a686
parent 313704 04bd4adb5ed3ca3e2571e8c15dd31ff7a1af3b8b
child 313706 40e59df093a7332f90a34de57a1d08766fc9a93c
push id20528
push userryanvm@gmail.com
push dateWed, 14 Sep 2016 00:48:16 +0000
treeherderfx-team@8a996d7c71a2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1286075
milestone51.0a1
Bug 1286075: add support for optimizing based on files changed in the push; r=gps MozReview-Commit-ID: 5di7TuL9X2P
taskcluster/taskgraph/files_changed.py
taskcluster/taskgraph/generator.py
taskcluster/taskgraph/optimize.py
taskcluster/taskgraph/task/base.py
taskcluster/taskgraph/task/docker_image.py
taskcluster/taskgraph/task/legacy.py
taskcluster/taskgraph/task/nightly_fennec.py
taskcluster/taskgraph/task/signing.py
taskcluster/taskgraph/task/transform.py
taskcluster/taskgraph/test/automationrelevance.json
taskcluster/taskgraph/test/test_files_changed.py
taskcluster/taskgraph/test/test_generator.py
taskcluster/taskgraph/test/test_optimize.py
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/files_changed.py
@@ -0,0 +1,65 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Support for optimizing tasks based on the set of files that have changed.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import requests
+from redo import retry
+from mozpack.path import match as mozpackmatch
+
+logger = logging.getLogger(__name__)
+_cache = {}
+
+
+def get_changed_files(repository, revision):
+    """
+    Get the set of files changed in the push headed by the given revision.
+    Responses are cached, so multiple calls with the same arguments are OK.
+    """
+    key = repository, revision
+    if key not in _cache:
+        url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision)
+        logger.debug("Querying version control for metadata: %s", url)
+
+        def get_automationrelevance():
+            response = requests.get(url, timeout=5)
+            return response.json()
+        contents = retry(get_automationrelevance, attempts=2, sleeptime=10)
+
+        logger.debug('{} commits influencing task scheduling:'
+                     .format(len(contents['changesets'])))
+        changed_files = set()
+        for c in contents['changesets']:
+            logger.debug(" {cset} {desc}".format(
+                cset=c['node'][0:12],
+                desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
+            changed_files |= set(c['files'])
+
+        _cache[key] = changed_files
+    return _cache[key]
+
+
+def check(params, file_patterns):
+    """Determine whether any of the files changed in the indicated push to
+    https://hg.mozilla.org match any of the given file patterns."""
+    repository = params.get('head_repository')
+    revision = params.get('head_rev')
+    if not repository or not revision:
+        logger.warning("Missing `head_repository` or `head_rev` parameters; "
+                       "assuming all files have changed")
+        return True
+
+    changed_files = get_changed_files(repository, revision)
+
+    for pattern in file_patterns:
+        for path in changed_files:
+            if mozpackmatch(path, pattern):
+                return True
+
+    return False
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -198,16 +198,17 @@ class TaskGraphGenerator(object):
             target_graph)
         yield 'target_task_graph', target_task_graph
 
         logger.info("Generating optimized task graph")
         do_not_optimize = set()
         if not self.parameters.get('optimize_target_tasks', True):
             do_not_optimize = target_task_set.graph.nodes
         optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
+                                                                    self.parameters,
                                                                     do_not_optimize)
         yield 'label_to_taskid', label_to_taskid
         yield 'optimized_task_graph', optimized_task_graph
 
     def _run_until(self, name):
         while name not in self._run_results:
             try:
                 k, v = self._run.next()
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -9,30 +9,31 @@ import re
 from .graph import Graph
 from .taskgraph import TaskGraph
 from slugid import nice as slugid
 
 logger = logging.getLogger(__name__)
 TASK_REFERENCE_PATTERN = re.compile('<([^>]+)>')
 
 
-def optimize_task_graph(target_task_graph, do_not_optimize, existing_tasks=None):
+def optimize_task_graph(target_task_graph, params, do_not_optimize, existing_tasks=None):
     """
     Perform task optimization, without optimizing tasks named in
     do_not_optimize.
     """
     named_links_dict = target_task_graph.graph.named_links_dict()
     label_to_taskid = {}
 
     # This proceeds in two phases.  First, mark all optimized tasks (those
     # which will be removed from the graph) as such, including a replacement
     # taskId where applicable.  Second, generate a new task graph containing
     # only the non-optimized tasks, with all task labels resolved to taskIds
     # and with task['dependencies'] populated.
     annotate_task_graph(target_task_graph=target_task_graph,
+                        params=params,
                         do_not_optimize=do_not_optimize,
                         named_links_dict=named_links_dict,
                         label_to_taskid=label_to_taskid,
                         existing_tasks=existing_tasks)
     return get_subgraph(target_task_graph, named_links_dict, label_to_taskid), label_to_taskid
 
 
 def resolve_task_references(label, task_def, taskid_for_edge_name):
@@ -54,17 +55,17 @@ def resolve_task_references(label, task_
                 return TASK_REFERENCE_PATTERN.sub(repl, val['task-reference'])
             else:
                 return {k: recurse(v) for k, v in val.iteritems()}
         else:
             return val
     return recurse(task_def)
 
 
-def annotate_task_graph(target_task_graph, do_not_optimize,
+def annotate_task_graph(target_task_graph, params, do_not_optimize,
                         named_links_dict, label_to_taskid, existing_tasks):
     """
     Annotate each task in the graph with .optimized (boolean) and .task_id
     (possibly None), following the rules for optimization and calling the task
     kinds' `optimize_task` method.
 
     As a side effect, label_to_taskid is updated with labels for all optimized
     tasks that are replaced with existing tasks.
@@ -89,17 +90,17 @@ def annotate_task_graph(target_task_grap
         if label in do_not_optimize:
             optimized = False
         # Let's check whether this task has been created before
         elif existing_tasks is not None and label in existing_tasks:
             optimized = True
             replacement_task_id = existing_tasks[label]
         # otherwise, examine the task itself (which may be an expensive operation)
         else:
-            optimized, replacement_task_id = task.optimize()
+            optimized, replacement_task_id = task.optimize(params)
 
         task.optimized = optimized
         task.task_id = replacement_task_id
         if replacement_task_id:
             label_to_taskid[label] = replacement_task_id
 
         if optimized:
             if replacement_task_id:
--- a/taskcluster/taskgraph/task/base.py
+++ b/taskcluster/taskgraph/task/base.py
@@ -74,17 +74,17 @@ class Task(object):
         """
         Get the set of task labels this task depends on, by querying the full
         task set, given as `taskgraph`.
 
         Returns a list of (task_label, dependency_name) pairs describing the
         dependencies.
         """
 
-    def optimize(self):
+    def optimize(self, params):
         """
         Determine whether this task can be optimized, and if it can, what taskId
         it should be replaced with.
 
         The return value is a tuple `(optimized, taskId)`.  If `optimized` is
         true, then the task will be optimized (in other words, not included in
         the task graph).  If the second argument is a taskid, then any
         dependencies on this task will isntead depend on that taskId.  It is an
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -113,17 +113,17 @@ class DockerImageTask(base.Task):
                              task=image_task['task'], attributes=attributes,
                              index_paths=index_paths))
 
         return tasks
 
     def get_dependencies(self, taskgraph):
         return []
 
-    def optimize(self):
+    def optimize(self, params):
         for index_path in self.index_paths:
             try:
                 url = INDEX_URL.format(index_path)
                 existing_task = json.load(urllib2.urlopen(url))
                 # Only return the task ID if the artifact exists for the indexed
                 # task.  Otherwise, continue on looking at each of the branches.  Method
                 # continues trying other branches in case mozilla-central has an expired
                 # artifact, but 'project' might not. Only return no task ID if all
--- a/taskcluster/taskgraph/task/legacy.py
+++ b/taskcluster/taskgraph/task/legacy.py
@@ -613,17 +613,17 @@ class LegacyTask(base.Task):
 
         # add a dependency on an image task, if needed
         if 'docker-image' in self.task_dict:
             deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
                          'docker-image'))
 
         return deps
 
-    def optimize(self):
+    def optimize(self, params):
         # no optimization for the moment
         return False, None
 
     @classmethod
     def from_json(cls, task_dict):
         legacy_task = cls(kind='legacy',
                           label=task_dict['label'],
                           attributes=task_dict['attributes'],
--- a/taskcluster/taskgraph/task/nightly_fennec.py
+++ b/taskcluster/taskgraph/task/nightly_fennec.py
@@ -104,10 +104,10 @@ class NightlyFennecTask(base.Task):
 
         # add a dependency on an image task, if needed
         if 'docker-image' in self.task_dict:
             deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
                          'docker-image'))
 
         return deps
 
-    def optimize(self):
+    def optimize(self, params):
         return False, None
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@@ -46,10 +46,10 @@ class SigningTask(base.Task):
             tasks.append(cls(kind, 'signing-nightly-fennec', task=task['task'],
                              attributes=attributes))
 
         return tasks
 
     def get_dependencies(self, taskgraph):
         return [('build-nightly-fennec', 'build-nightly-fennec')]
 
-    def optimize(self):
+    def optimize(self, params):
         return False, None
--- a/taskcluster/taskgraph/task/transform.py
+++ b/taskcluster/taskgraph/task/transform.py
@@ -2,29 +2,30 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
 
 from . import base
+from .. import files_changed
 from ..util.python_path import find_object
 from ..util.yaml import load_yaml
 from ..transforms.base import TransformSequence, TransformConfig
 
 logger = logging.getLogger(__name__)
 
 
 class TransformTask(base.Task):
     """
     Tasks of this class are generated by applying transformations to a sequence
     of input entities.  By default, it gets those inputs from YAML data in the
-    kind directory, but subclasses may override `get_inputs` to produce them
-    in some other way.
+    kind directory, but subclasses may override `get_inputs` to produce them in
+    some other way.
     """
 
     @classmethod
     def get_inputs(cls, kind, path, config, params, loaded_tasks):
         """
         Get the input elements that will be transformed into tasks.  The
         elements themselves are free-form, and become the input to the first
         transform.
@@ -58,16 +59,24 @@ class TransformTask(base.Task):
 
         # perform the transformations
         trans_config = TransformConfig(kind, path, config, params)
         tasks = [cls(kind, t) for t in transforms(trans_config, inputs)]
         return tasks
 
     def __init__(self, kind, task):
         self.dependencies = task['dependencies']
+        self.when = task['when']
         super(TransformTask, self).__init__(kind, task['label'],
                                             task['attributes'], task['task'])
 
     def get_dependencies(self, taskgraph):
         return [(label, name) for name, label in self.dependencies.items()]
 
-    def optimize(self):
+    def optimize(self, params):
+        if 'files-changed' in self.when:
+            changed = files_changed.check(
+                params, self.when['files-changed'])
+            if not changed:
+                logger.debug('no files found matching a pattern in `when.files-changed` for '
+                             + self.label)
+                return True, None
         return False, None
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/automationrelevance.json
@@ -0,0 +1,425 @@
+{
+    "changesets": [
+        {
+            "author": "James Long <longster@gmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1300866",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866"
+                }
+            ],
+            "date": [
+                1473196655.0,
+                14400
+            ],
+            "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/debugger/new/index.html"
+            ],
+            "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "parents": [
+                "37c9349b4e8167a61b08b7e119c21ea177b98942"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312890,
+            "reviewers": [
+                {
+                    "name": "jlast",
+                    "revset": "reviewer(jlast)"
+                },
+                {
+                    "name": "bgrins",
+                    "revset": "reviewer(bgrins)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Wes Kocher <wkocher@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [],
+            "date": [
+                1473208638.0,
+                25200
+            ],
+            "desc": "Merge m-c to fx-team, a=merge",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "taskcluster/scripts/builder/build-l10n.sh"
+            ],
+            "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+            "parents": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312891,
+            "reviewers": [
+                {
+                    "name": "merge",
+                    "revset": "reviewer(merge)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Towkir Ahmed <towkir17@gmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1296648",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648"
+                }
+            ],
+            "date": [
+                1472957580.0,
+                14400
+            ],
+            "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/themes/rules.css"
+            ],
+            "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+            "parents": [
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312892,
+            "reviewers": [
+                {
+                    "name": "ntim",
+                    "revset": "reviewer(ntim)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Oriol <oriol-bugzilla@hotmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1300336",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336"
+                }
+            ],
+            "date": [
+                1472921160.0,
+                14400
+            ],
+            "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/webconsole/test/browser_webconsole_output_06.js",
+                "devtools/server/actors/object.js"
+            ],
+            "node": "99c542fa43a72ee863c813b5624048d1b443549b",
+            "parents": [
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312893,
+            "reviewers": [
+                {
+                    "name": "fitzgen",
+                    "revset": "reviewer(fitzgen)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Ruturaj Vartak <ruturaj@gmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1295010",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
+                }
+            ],
+            "date": [
+                1472854020.0,
+                -7200
+            ],
+            "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK",
+            "extra": {
+                "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd",
+                "branch": "default",
+                "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba",
+                "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651"
+            },
+            "files": [
+                "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
+                "devtools/client/inspector/test/head.js",
+                "devtools/server/actors/highlighters/eye-dropper.js"
+            ],
+            "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+            "parents": [
+                "99c542fa43a72ee863c813b5624048d1b443549b"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312894,
+            "reviewers": [
+                {
+                    "name": "pbro",
+                    "revset": "reviewer(pbro)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Matteo Ferretti <mferretti@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1299154",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154"
+                }
+            ],
+            "date": [
+                1472629906.0,
+                -7200
+            ],
+            "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n",
+            "extra": {
+                "branch": "default",
+                "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200"
+            },
+            "files": [
+                "docshell/base/nsDocShell.cpp",
+                "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html"
+            ],
+            "node": "541c9086c0f27fba60beecc9bc94543103895c86",
+            "parents": [
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312895,
+            "reviewers": [
+                {
+                    "name": "mstange",
+                    "revset": "reviewer(mstange)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Patrick Brosset <pbrosset@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1295010",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
+                }
+            ],
+            "date": [
+                1473239449.0,
+                -7200
+            ],
+            "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
+                "devtools/client/inspector/test/head.js"
+            ],
+            "node": "041a925171e431bf51fb50193ab19d156088c89a",
+            "parents": [
+                "541c9086c0f27fba60beecc9bc94543103895c86"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312896,
+            "reviewers": [
+                {
+                    "name": "me",
+                    "revset": "reviewer(me)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [],
+            "date": [
+                1473261233.0,
+                -7200
+            ],
+            "desc": "merge fx-team to mozilla-central a=merge",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [],
+            "node": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "parents": [
+                "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9",
+                "041a925171e431bf51fb50193ab19d156088c89a"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312897,
+            "reviewers": [
+                {
+                    "name": "merge",
+                    "revset": "reviewer(merge)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        }
+    ],
+    "visible": true
+}
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_files_changed.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import os
+import unittest
+
+from .. import files_changed
+
+PARAMS = {
+    'head_repository': 'https://hg.mozilla.org/mozilla-central',
+    'head_rev': 'a14f88a9af7a',
+}
+
+FILES_CHANGED = [
+    'devtools/client/debugger/new/index.html',
+    'devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js',
+    'devtools/client/inspector/test/head.js',
+    'devtools/client/themes/rules.css',
+    'devtools/client/webconsole/test/browser_webconsole_output_06.js',
+    'devtools/server/actors/highlighters/eye-dropper.js',
+    'devtools/server/actors/object.js',
+    'docshell/base/nsDocShell.cpp',
+    'dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html',
+    'taskcluster/scripts/builder/build-l10n.sh',
+]
+
+
+class FakeResponse:
+
+    def json(self):
+        with open(os.path.join(os.path.dirname(__file__), 'automationrelevance.json')) as f:
+            return json.load(f)
+
+
+class TestGetChangedFiles(unittest.TestCase):
+
+    def setUp(self):
+        files_changed._cache.clear()
+        self.old_get = files_changed.requests.get
+
+        def fake_get(url, **kwargs):
+            return FakeResponse()
+        files_changed.requests.get = fake_get
+
+    def tearDown(self):
+        files_changed.requests.get = self.old_get
+
+    def test_get_changed_files(self):
+        """Get_changed_files correctly gets the list of changed files in a push.
+        This tests against the production hg.mozilla.org so that it will detect
+        any changes in the format of the returned data."""
+        self.assertEqual(
+            sorted(files_changed.get_changed_files(PARAMS['head_repository'], PARAMS['head_rev'])),
+            FILES_CHANGED)
+
+
+class TestCheck(unittest.TestCase):
+
+    def setUp(self):
+        files_changed._cache[PARAMS['head_repository'], PARAMS['head_rev']] = FILES_CHANGED
+
+    def test_check_no_params(self):
+        self.assertTrue(files_changed.check({}, ["ignored"]))
+
+    def test_check_no_match(self):
+        self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"]))
+
+    def test_check_match(self):
+        self.assertTrue(files_changed.check(PARAMS, ["devtools/**"]))
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -29,17 +29,17 @@ class FakeTask(base.Task):
 
     def get_dependencies(self, full_task_set):
         i = self.i
         if i > 0:
             return [('{}-t-{}'.format(self.kind, i - 1), 'prev')]
         else:
             return []
 
-    def optimize(self):
+    def optimize(self, params):
         return False, None
 
 
 class FakeKind(Kind):
 
     def _get_impl_class(self):
         return FakeTask
 
--- a/taskcluster/taskgraph/test/test_optimize.py
+++ b/taskcluster/taskgraph/test/test_optimize.py
@@ -80,85 +80,85 @@ class TestOptimize(unittest.TestCase):
             return 'SLUGID' if task_id and len(task_id) == 22 else task_id
         got_annotations = {
             t.label: (t.optimized, repl(t.task_id)) for t in graph.tasks.itervalues()
         }
         self.assertEqual(got_annotations, annotations)
 
     def test_annotate_task_graph_no_optimize(self):
         "annotating marks everything as un-optimized if the kind returns that"
-        OptimizingTask.optimize = lambda self: (False, None)
+        OptimizingTask.optimize = lambda self, params: (False, None)
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             self.make_task('task3'),
             ('task2', 'task1', 'build'),
             ('task2', 'task3', 'image'),
         )
-        annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
+        annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
         self.assert_annotations(
             graph,
             task1=(False, None),
             task2=(False, None),
             task3=(False, None)
         )
 
     def test_annotate_task_graph_taskid_without_optimize(self):
         "raises exception if kind returns a taskid without optimizing"
-        OptimizingTask.optimize = lambda self: (False, 'some-taskid')
+        OptimizingTask.optimize = lambda self, params: (False, 'some-taskid')
         graph = self.make_graph(self.make_task('task1'))
         self.assertRaises(
             Exception,
-            lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
+            lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
         )
 
     def test_annotate_task_graph_optimize_away_dependency(self):
         "raises exception if kind optimizes away a task on which another depends"
         OptimizingTask.optimize = \
-            lambda self: (True, None) if self.label == 'task1' else (False, None)
+            lambda self, params: (True, None) if self.label == 'task1' else (False, None)
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             ('task2', 'task1', 'build'),
         )
         self.assertRaises(
             Exception,
-            lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
+            lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
         )
 
     def test_annotate_task_graph_do_not_optimize(self):
         "annotating marks everything as un-optimized if in do_not_optimize"
-        OptimizingTask.optimize = lambda self: (True, 'taskid')
+        OptimizingTask.optimize = lambda self, params: (True, 'taskid')
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             ('task2', 'task1', 'build'),
         )
         label_to_taskid = {}
-        annotate_task_graph(graph, {'task1', 'task2'},
+        annotate_task_graph(graph, {}, {'task1', 'task2'},
                             graph.graph.named_links_dict(), label_to_taskid, None)
         self.assert_annotations(
             graph,
             task1=(False, None),
             task2=(False, None)
         )
         self.assertEqual
 
     def test_annotate_task_graph_nos_do_not_propagate(self):
         "a task with a non-optimized dependency can be optimized"
         OptimizingTask.optimize = \
-            lambda self: (False, None) if self.label == 'task1' else (True, 'taskid')
+            lambda self, params: (False, None) if self.label == 'task1' else (True, 'taskid')
         graph = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             self.make_task('task3'),
             ('task2', 'task1', 'build'),
             ('task2', 'task3', 'image'),
         )
-        annotate_task_graph(graph, set(),
+        annotate_task_graph(graph, {}, set(),
                             graph.graph.named_links_dict(), {}, None)
         self.assert_annotations(
             graph,
             task1=(False, None),
             task2=(True, 'taskid'),
             task3=(True, 'taskid')
         )
 
@@ -237,20 +237,20 @@ class TestOptimize(unittest.TestCase):
         self.assertEqual(sub.tasks[task2].task_id, task2)
         self.assertEqual(sorted(sub.tasks[task2].task['dependencies']), sorted([task3, 'dep1']))
         self.assertEqual(sub.tasks[task2].task['payload'], 'http://dep1/' + task3)
         self.assertEqual(sub.tasks[task3].task_id, task3)
 
     def test_optimize(self):
         "optimize_task_graph annotates and extracts the subgraph from a simple graph"
         OptimizingTask.optimize = \
-            lambda self: (True, 'dep1') if self.label == 'task1' else (False, None)
+            lambda self, params: (True, 'dep1') if self.label == 'task1' else (False, None)
         input = self.make_graph(
             self.make_task('task1'),
             self.make_task('task2'),
             self.make_task('task3'),
             ('task2', 'task1', 'build'),
             ('task2', 'task3', 'image'),
         )
-        opt, label_to_taskid = optimize_task_graph(input, set())
+        opt, label_to_taskid = optimize_task_graph(input, {}, set())
         self.assertEqual(opt.graph, graph.Graph(
             {label_to_taskid['task2'], label_to_taskid['task3']},
             {(label_to_taskid['task2'], label_to_taskid['task3'], 'image')}))
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -49,16 +49,17 @@ job_description_schema = Schema({
     Optional('scopes'): task_description_schema['scopes'],
     Optional('extra'): task_description_schema['extra'],
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
     Optional('coalesce-name'): task_description_schema['coalesce-name'],
     Optional('worker-type'): task_description_schema['worker-type'],
     Required('worker'): task_description_schema['worker'],
+    Optional('when'): task_description_schema['when'],
 
     # A description of how to run this job.
     'run': {
         # The key to a job implementation in a peer module to this one
         'using': basestring,
 
         # Any remaining content is verified against that job implementation's
         # own schema.
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -207,16 +207,26 @@ task_description_schema = Schema({
             Optional('repository'): basestring,
             Optional('project'): basestring,
         },
         'properties': {
             'product': basestring,
             Extra: basestring,  # additional properties are allowed
         },
     }),
+
+    # The "when" section contains descriptions of the circumstances
+    # under which this task can be "optimized", that is, left out of the
+    # task graph because it is unnecessary.
+    Optional('when'): Any({
+        # This task only needs to be run if a file matching one of the given
+        # patterns has changed in the push.  The patterns use the mozpack
+        # match function (python/mozbuild/mozpack/path.py).
+        Optional('files-changed'): [basestring],
+    }),
 })
 
 GROUP_NAMES = {
     'tc': 'Executed by TaskCluster',
     'tc-e10s': 'Executed by TaskCluster with e10s',
     'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster',
     'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s',
     'tc-Fxfn-r': 'Firefox functional tests (remote) executed by TaskCluster',