author | Dustin J. Mitchell <dustin@mozilla.com> |
Mon, 12 Sep 2016 18:40:12 +0000 | |
changeset 313783 | 315bae6d1488fa36f2f90501218159241866a686 |
parent 313782 | 04bd4adb5ed3ca3e2571e8c15dd31ff7a1af3b8b |
child 313784 | 40e59df093a7332f90a34de57a1d08766fc9a93c |
push id | 32255 |
push user | ryanvm@gmail.com |
push date | Wed, 14 Sep 2016 00:47:02 +0000 |
treeherder | autoland@d8f95b350aa2 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | gps |
bugs | 1286075 |
milestone | 51.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
new file mode 100644 --- /dev/null +++ b/taskcluster/taskgraph/files_changed.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Support for optimizing tasks based on the set of files that have changed. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import logging +import requests +from redo import retry +from mozpack.path import match as mozpackmatch + +logger = logging.getLogger(__name__) +_cache = {} + + +def get_changed_files(repository, revision): + """ + Get the set of files changed in the push headed by the given revision. + Responses are cached, so multiple calls with the same arguments are OK. + """ + key = repository, revision + if key not in _cache: + url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision) + logger.debug("Querying version control for metadata: %s", url) + + def get_automationrelevance(): + response = requests.get(url, timeout=5) + return response.json() + contents = retry(get_automationrelevance, attempts=2, sleeptime=10) + + logger.debug('{} commits influencing task scheduling:' + .format(len(contents['changesets']))) + changed_files = set() + for c in contents['changesets']: + logger.debug(" {cset} {desc}".format( + cset=c['node'][0:12], + desc=c['desc'].splitlines()[0].encode('ascii', 'ignore'))) + changed_files |= set(c['files']) + + _cache[key] = changed_files + return _cache[key] + + +def check(params, file_patterns): + """Determine whether any of the files changed in the indicated push to + https://hg.mozilla.org match any of the given file patterns.""" + repository = params.get('head_repository') + revision = params.get('head_rev') + if not repository or not revision: + logger.warning("Missing `head_repository` or `head_rev` parameters; " + "assuming all files have changed") + return True + + changed_files = get_changed_files(repository, revision) + + for pattern in file_patterns: + for path in changed_files: + if mozpackmatch(path, pattern): + return True + + return False
--- a/taskcluster/taskgraph/generator.py +++ b/taskcluster/taskgraph/generator.py @@ -198,16 +198,17 @@ class TaskGraphGenerator(object): target_graph) yield 'target_task_graph', target_task_graph logger.info("Generating optimized task graph") do_not_optimize = set() if not self.parameters.get('optimize_target_tasks', True): do_not_optimize = target_task_set.graph.nodes optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph, + self.parameters, do_not_optimize) yield 'label_to_taskid', label_to_taskid yield 'optimized_task_graph', optimized_task_graph def _run_until(self, name): while name not in self._run_results: try: k, v = self._run.next()
--- a/taskcluster/taskgraph/optimize.py +++ b/taskcluster/taskgraph/optimize.py @@ -9,30 +9,31 @@ import re from .graph import Graph from .taskgraph import TaskGraph from slugid import nice as slugid logger = logging.getLogger(__name__) TASK_REFERENCE_PATTERN = re.compile('<([^>]+)>') -def optimize_task_graph(target_task_graph, do_not_optimize, existing_tasks=None): +def optimize_task_graph(target_task_graph, params, do_not_optimize, existing_tasks=None): """ Perform task optimization, without optimizing tasks named in do_not_optimize. """ named_links_dict = target_task_graph.graph.named_links_dict() label_to_taskid = {} # This proceeds in two phases. First, mark all optimized tasks (those # which will be removed from the graph) as such, including a replacement # taskId where applicable. Second, generate a new task graph containing # only the non-optimized tasks, with all task labels resolved to taskIds # and with task['dependencies'] populated. annotate_task_graph(target_task_graph=target_task_graph, + params=params, do_not_optimize=do_not_optimize, named_links_dict=named_links_dict, label_to_taskid=label_to_taskid, existing_tasks=existing_tasks) return get_subgraph(target_task_graph, named_links_dict, label_to_taskid), label_to_taskid def resolve_task_references(label, task_def, taskid_for_edge_name): @@ -54,17 +55,17 @@ def resolve_task_references(label, task_ return TASK_REFERENCE_PATTERN.sub(repl, val['task-reference']) else: return {k: recurse(v) for k, v in val.iteritems()} else: return val return recurse(task_def) -def annotate_task_graph(target_task_graph, do_not_optimize, +def annotate_task_graph(target_task_graph, params, do_not_optimize, named_links_dict, label_to_taskid, existing_tasks): """ Annotate each task in the graph with .optimized (boolean) and .task_id (possibly None), following the rules for optimization and calling the task kinds' `optimize_task` method. As a side effect, label_to_taskid is updated with labels for all optimized tasks that are replaced with existing tasks. @@ -89,17 +90,17 @@ def annotate_task_graph(target_task_grap if label in do_not_optimize: optimized = False # Let's check whether this task has been created before elif existing_tasks is not None and label in existing_tasks: optimized = True replacement_task_id = existing_tasks[label] # otherwise, examine the task itself (which may be an expensive operation) else: - optimized, replacement_task_id = task.optimize() + optimized, replacement_task_id = task.optimize(params) task.optimized = optimized task.task_id = replacement_task_id if replacement_task_id: label_to_taskid[label] = replacement_task_id if optimized: if replacement_task_id:
--- a/taskcluster/taskgraph/task/base.py +++ b/taskcluster/taskgraph/task/base.py @@ -74,17 +74,17 @@ class Task(object): """ Get the set of task labels this task depends on, by querying the full task set, given as `taskgraph`. Returns a list of (task_label, dependency_name) pairs describing the dependencies. """ - def optimize(self): + def optimize(self, params): """ Determine whether this task can be optimized, and if it can, what taskId it should be replaced with. The return value is a tuple `(optimized, taskId)`. If `optimized` is true, then the task will be optimized (in other words, not included in the task graph). If the second argument is a taskid, then any dependencies on this task will isntead depend on that taskId. It is an
--- a/taskcluster/taskgraph/task/docker_image.py +++ b/taskcluster/taskgraph/task/docker_image.py @@ -113,17 +113,17 @@ class DockerImageTask(base.Task): task=image_task['task'], attributes=attributes, index_paths=index_paths)) return tasks def get_dependencies(self, taskgraph): return [] - def optimize(self): + def optimize(self, params): for index_path in self.index_paths: try: url = INDEX_URL.format(index_path) existing_task = json.load(urllib2.urlopen(url)) # Only return the task ID if the artifact exists for the indexed # task. Otherwise, continue on looking at each of the branches. Method # continues trying other branches in case mozilla-central has an expired # artifact, but 'project' might not. Only return no task ID if all
--- a/taskcluster/taskgraph/task/legacy.py +++ b/taskcluster/taskgraph/task/legacy.py @@ -613,17 +613,17 @@ class LegacyTask(base.Task): # add a dependency on an image task, if needed if 'docker-image' in self.task_dict: deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict), 'docker-image')) return deps - def optimize(self): + def optimize(self, params): # no optimization for the moment return False, None @classmethod def from_json(cls, task_dict): legacy_task = cls(kind='legacy', label=task_dict['label'], attributes=task_dict['attributes'],
--- a/taskcluster/taskgraph/task/nightly_fennec.py +++ b/taskcluster/taskgraph/task/nightly_fennec.py @@ -104,10 +104,10 @@ class NightlyFennecTask(base.Task): # add a dependency on an image task, if needed if 'docker-image' in self.task_dict: deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict), 'docker-image')) return deps - def optimize(self): + def optimize(self, params): return False, None
--- a/taskcluster/taskgraph/task/signing.py +++ b/taskcluster/taskgraph/task/signing.py @@ -46,10 +46,10 @@ class SigningTask(base.Task): tasks.append(cls(kind, 'signing-nightly-fennec', task=task['task'], attributes=attributes)) return tasks def get_dependencies(self, taskgraph): return [('build-nightly-fennec', 'build-nightly-fennec')] - def optimize(self): + def optimize(self, params): return False, None
--- a/taskcluster/taskgraph/task/transform.py +++ b/taskcluster/taskgraph/task/transform.py @@ -2,29 +2,30 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, print_function, unicode_literals import logging from . import base +from .. import files_changed from ..util.python_path import find_object from ..util.yaml import load_yaml from ..transforms.base import TransformSequence, TransformConfig logger = logging.getLogger(__name__) class TransformTask(base.Task): """ Tasks of this class are generated by applying transformations to a sequence of input entities. By default, it gets those inputs from YAML data in the - kind directory, but subclasses may override `get_inputs` to produce them - in some other way. + kind directory, but subclasses may override `get_inputs` to produce them in + some other way. """ @classmethod def get_inputs(cls, kind, path, config, params, loaded_tasks): """ Get the input elements that will be transformed into tasks. The elements themselves are free-form, and become the input to the first transform. @@ -58,16 +59,24 @@ class TransformTask(base.Task): # perform the transformations trans_config = TransformConfig(kind, path, config, params) tasks = [cls(kind, t) for t in transforms(trans_config, inputs)] return tasks def __init__(self, kind, task): self.dependencies = task['dependencies'] + self.when = task['when'] super(TransformTask, self).__init__(kind, task['label'], task['attributes'], task['task']) def get_dependencies(self, taskgraph): return [(label, name) for name, label in self.dependencies.items()] - def optimize(self): + def optimize(self, params): + if 'files-changed' in self.when: + changed = files_changed.check( + params, self.when['files-changed']) + if not changed: + logger.debug('no files found matching a pattern in `when.files-changed` for ' + + self.label) + return True, None return False, None
new file mode 100644 --- /dev/null +++ b/taskcluster/taskgraph/test/automationrelevance.json @@ -0,0 +1,425 @@ +{ + "changesets": [ + { + "author": "James Long <longster@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1300866", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866" + } + ], + "date": [ + 1473196655.0, + 14400 + ], + "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/debugger/new/index.html" + ], + "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "parents": [ + "37c9349b4e8167a61b08b7e119c21ea177b98942" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312890, + "reviewers": [ + { + "name": "jlast", + "revset": "reviewer(jlast)" + }, + { + "name": "bgrins", + "revset": "reviewer(bgrins)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Wes Kocher <wkocher@mozilla.com>", + "backsoutnodes": [], + "bugs": [], + "date": [ + 1473208638.0, + 25200 + ], + "desc": "Merge m-c to fx-team, a=merge", + "extra": { + "branch": "default" + }, + "files": [ + "taskcluster/scripts/builder/build-l10n.sh" + ], + "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "parents": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312891, + "reviewers": [ + { + "name": "merge", + "revset": "reviewer(merge)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Towkir Ahmed <towkir17@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1296648", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648" + } + ], + "date": [ + 1472957580.0, + 14400 + ], + "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/themes/rules.css" + ], + "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "parents": [ + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312892, + "reviewers": [ + { + "name": "ntim", + "revset": "reviewer(ntim)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Oriol <oriol-bugzilla@hotmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1300336", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336" + } + ], + "date": [ + 1472921160.0, + 14400 + ], + "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/webconsole/test/browser_webconsole_output_06.js", + "devtools/server/actors/object.js" + ], + "node": "99c542fa43a72ee863c813b5624048d1b443549b", + "parents": [ + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312893, + "reviewers": [ + { + "name": "fitzgen", + "revset": "reviewer(fitzgen)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Ruturaj Vartak <ruturaj@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1295010", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010" + } + ], + "date": [ + 1472854020.0, + -7200 + ], + "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK", + "extra": { + "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd", + "branch": "default", + "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba", + "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651" + }, + "files": [ + "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js", + "devtools/client/inspector/test/head.js", + "devtools/server/actors/highlighters/eye-dropper.js" + ], + "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "parents": [ + "99c542fa43a72ee863c813b5624048d1b443549b" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312894, + "reviewers": [ + { + "name": "pbro", + "revset": "reviewer(pbro)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Matteo Ferretti <mferretti@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1299154", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154" + } + ], + "date": [ + 1472629906.0, + -7200 + ], + "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n", + "extra": { + "branch": "default", + "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200" + }, + "files": [ + "docshell/base/nsDocShell.cpp", + "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html" + ], + "node": "541c9086c0f27fba60beecc9bc94543103895c86", + "parents": [ + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312895, + "reviewers": [ + { + "name": "mstange", + "revset": "reviewer(mstange)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Patrick Brosset <pbrosset@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1295010", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010" + } + ], + "date": [ + 1473239449.0, + -7200 + ], + "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js", + "devtools/client/inspector/test/head.js" + ], + "node": "041a925171e431bf51fb50193ab19d156088c89a", + "parents": [ + "541c9086c0f27fba60beecc9bc94543103895c86" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312896, + "reviewers": [ + { + "name": "me", + "revset": "reviewer(me)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>", + "backsoutnodes": [], + "bugs": [], + "date": [ + 1473261233.0, + -7200 + ], + "desc": "merge fx-team to mozilla-central a=merge", + "extra": { + "branch": "default" + }, + "files": [], + "node": "a14f88a9af7a59e677478694bafd9375ac53683e", + "parents": [ + "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9", + "041a925171e431bf51fb50193ab19d156088c89a" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312897, + "reviewers": [ + { + "name": "merge", + "revset": "reviewer(merge)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + } + ], + "visible": true +} +
new file mode 100644 --- /dev/null +++ b/taskcluster/taskgraph/test/test_files_changed.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import json +import os +import unittest + +from .. import files_changed + +PARAMS = { + 'head_repository': 'https://hg.mozilla.org/mozilla-central', + 'head_rev': 'a14f88a9af7a', +} + +FILES_CHANGED = [ + 'devtools/client/debugger/new/index.html', + 'devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js', + 'devtools/client/inspector/test/head.js', + 'devtools/client/themes/rules.css', + 'devtools/client/webconsole/test/browser_webconsole_output_06.js', + 'devtools/server/actors/highlighters/eye-dropper.js', + 'devtools/server/actors/object.js', + 'docshell/base/nsDocShell.cpp', + 'dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html', + 'taskcluster/scripts/builder/build-l10n.sh', +] + + +class FakeResponse: + + def json(self): + with open(os.path.join(os.path.dirname(__file__), 'automationrelevance.json')) as f: + return json.load(f) + + +class TestGetChangedFiles(unittest.TestCase): + + def setUp(self): + files_changed._cache.clear() + self.old_get = files_changed.requests.get + + def fake_get(url, **kwargs): + return FakeResponse() + files_changed.requests.get = fake_get + + def tearDown(self): + files_changed.requests.get = self.old_get + + def test_get_changed_files(self): + """Get_changed_files correctly gets the list of changed files in a push. + This tests against the production hg.mozilla.org so that it will detect + any changes in the format of the returned data.""" + self.assertEqual( + sorted(files_changed.get_changed_files(PARAMS['head_repository'], PARAMS['head_rev'])), + FILES_CHANGED) + + +class TestCheck(unittest.TestCase): + + def setUp(self): + files_changed._cache[PARAMS['head_repository'], PARAMS['head_rev']] = FILES_CHANGED + + def test_check_no_params(self): + self.assertTrue(files_changed.check({}, ["ignored"])) + + def test_check_no_match(self): + self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"])) + + def test_check_match(self): + self.assertTrue(files_changed.check(PARAMS, ["devtools/**"]))
--- a/taskcluster/taskgraph/test/test_generator.py +++ b/taskcluster/taskgraph/test/test_generator.py @@ -29,17 +29,17 @@ class FakeTask(base.Task): def get_dependencies(self, full_task_set): i = self.i if i > 0: return [('{}-t-{}'.format(self.kind, i - 1), 'prev')] else: return [] - def optimize(self): + def optimize(self, params): return False, None class FakeKind(Kind): def _get_impl_class(self): return FakeTask
--- a/taskcluster/taskgraph/test/test_optimize.py +++ b/taskcluster/taskgraph/test/test_optimize.py @@ -80,85 +80,85 @@ class TestOptimize(unittest.TestCase): return 'SLUGID' if task_id and len(task_id) == 22 else task_id got_annotations = { t.label: (t.optimized, repl(t.task_id)) for t in graph.tasks.itervalues() } self.assertEqual(got_annotations, annotations) def test_annotate_task_graph_no_optimize(self): "annotating marks everything as un-optimized if the kind returns that" - OptimizingTask.optimize = lambda self: (False, None) + OptimizingTask.optimize = lambda self, params: (False, None) graph = self.make_graph( self.make_task('task1'), self.make_task('task2'), self.make_task('task3'), ('task2', 'task1', 'build'), ('task2', 'task3', 'image'), ) - annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None) + annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None) self.assert_annotations( graph, task1=(False, None), task2=(False, None), task3=(False, None) ) def test_annotate_task_graph_taskid_without_optimize(self): "raises exception if kind returns a taskid without optimizing" - OptimizingTask.optimize = lambda self: (False, 'some-taskid') + OptimizingTask.optimize = lambda self, params: (False, 'some-taskid') graph = self.make_graph(self.make_task('task1')) self.assertRaises( Exception, - lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None) + lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None) ) def test_annotate_task_graph_optimize_away_dependency(self): "raises exception if kind optimizes away a task on which another depends" OptimizingTask.optimize = \ - lambda self: (True, None) if self.label == 'task1' else (False, None) + lambda self, params: (True, None) if self.label == 'task1' else (False, None) graph = self.make_graph( self.make_task('task1'), self.make_task('task2'), ('task2', 'task1', 'build'), ) self.assertRaises( Exception, - lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None) + lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None) ) def test_annotate_task_graph_do_not_optimize(self): "annotating marks everything as un-optimized if in do_not_optimize" - OptimizingTask.optimize = lambda self: (True, 'taskid') + OptimizingTask.optimize = lambda self, params: (True, 'taskid') graph = self.make_graph( self.make_task('task1'), self.make_task('task2'), ('task2', 'task1', 'build'), ) label_to_taskid = {} - annotate_task_graph(graph, {'task1', 'task2'}, + annotate_task_graph(graph, {}, {'task1', 'task2'}, graph.graph.named_links_dict(), label_to_taskid, None) self.assert_annotations( graph, task1=(False, None), task2=(False, None) ) self.assertEqual def test_annotate_task_graph_nos_do_not_propagate(self): "a task with a non-optimized dependency can be optimized" OptimizingTask.optimize = \ - lambda self: (False, None) if self.label == 'task1' else (True, 'taskid') + lambda self, params: (False, None) if self.label == 'task1' else (True, 'taskid') graph = self.make_graph( self.make_task('task1'), self.make_task('task2'), self.make_task('task3'), ('task2', 'task1', 'build'), ('task2', 'task3', 'image'), ) - annotate_task_graph(graph, set(), + annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None) self.assert_annotations( graph, task1=(False, None), task2=(True, 'taskid'), task3=(True, 'taskid') ) @@ -237,20 +237,20 @@ class TestOptimize(unittest.TestCase): self.assertEqual(sub.tasks[task2].task_id, task2) self.assertEqual(sorted(sub.tasks[task2].task['dependencies']), sorted([task3, 'dep1'])) self.assertEqual(sub.tasks[task2].task['payload'], 'http://dep1/' + task3) self.assertEqual(sub.tasks[task3].task_id, task3) def test_optimize(self): "optimize_task_graph annotates and extracts the subgraph from a simple graph" OptimizingTask.optimize = \ - lambda self: (True, 'dep1') if self.label == 'task1' else (False, None) + lambda self, params: (True, 'dep1') if self.label == 'task1' else (False, None) input = self.make_graph( self.make_task('task1'), self.make_task('task2'), self.make_task('task3'), ('task2', 'task1', 'build'), ('task2', 'task3', 'image'), ) - opt, label_to_taskid = optimize_task_graph(input, set()) + opt, label_to_taskid = optimize_task_graph(input, {}, set()) self.assertEqual(opt.graph, graph.Graph( {label_to_taskid['task2'], label_to_taskid['task3']}, {(label_to_taskid['task2'], label_to_taskid['task3'], 'image')}))
--- a/taskcluster/taskgraph/transforms/job/__init__.py +++ b/taskcluster/taskgraph/transforms/job/__init__.py @@ -49,16 +49,17 @@ job_description_schema = Schema({ Optional('scopes'): task_description_schema['scopes'], Optional('extra'): task_description_schema['extra'], Optional('treeherder'): task_description_schema['treeherder'], Optional('index'): task_description_schema['index'], Optional('run-on-projects'): task_description_schema['run-on-projects'], Optional('coalesce-name'): task_description_schema['coalesce-name'], Optional('worker-type'): task_description_schema['worker-type'], Required('worker'): task_description_schema['worker'], + Optional('when'): task_description_schema['when'], # A description of how to run this job. 'run': { # The key to a job implementation in a peer module to this one 'using': basestring, # Any remaining content is verified against that job implementation's # own schema.
--- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -207,16 +207,26 @@ task_description_schema = Schema({ Optional('repository'): basestring, Optional('project'): basestring, }, 'properties': { 'product': basestring, Extra: basestring, # additional properties are allowed }, }), + + # The "when" section contains descriptions of the circumstances + # under which this task can be "optimized", that is, left out of the + # task graph because it is unnecessary. + Optional('when'): Any({ + # This task only needs to be run if a file matching one of the given + # patterns has changed in the push. The patterns use the mozpack + # match function (python/mozbuild/mozpack/path.py). + Optional('files-changed'): [basestring], + }), }) GROUP_NAMES = { 'tc': 'Executed by TaskCluster', 'tc-e10s': 'Executed by TaskCluster with e10s', 'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster', 'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s', 'tc-Fxfn-r': 'Firefox functional tests (remote) executed by TaskCluster',