Bug 1383880: add only-if-dependencies-run optimization for follow-ons; r=ahal
☠☠ backed out by 1d02aa80f2f8 ☠ ☠
authorDustin J. Mitchell <dustin@mozilla.com>
Fri, 25 Aug 2017 21:18:13 +0000
changeset 378306 53f5d47a7cb0f32f7cc1281f8ca5ce5f855e3fc4
parent 378305 a0abda41172ab4646897ada3a8e993369de3d87d
child 378307 120bb4c9f8aaf17fbb50d21573d778a705fe303a
push id32428
push userarchaeopteryx@coole-files.de
push dateSat, 02 Sep 2017 08:52:28 +0000
treeherdermozilla-central@b01a7e57425b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1383880: add only-if-dependencies-run optimization for follow-ons; r=ahal MozReview-Commit-ID: JuhwyQIx3Mh
--- a/taskcluster/ci/upload-generated-sources/kind.yml
+++ b/taskcluster/ci/upload-generated-sources/kind.yml
@@ -26,10 +26,12 @@ job-template:
      docker-image: {in-tree: "lint"}
      max-run-time: 600
     using: run-task
     command: >
             cd /builds/worker/checkouts/gecko &&
             ./mach python build/upload_generated_sources.py ${ARTIFACT_URL}
+  optimization:
+    only-if-dependencies-run: null
       - secrets:get:project/releng/gecko/build/level-{level}/gecko-generated-sources-upload
--- a/taskcluster/ci/upload-symbols/kind.yml
+++ b/taskcluster/ci/upload-symbols/kind.yml
@@ -38,10 +38,12 @@ job-template:
        os: linux
        max-run-time: 600
        command: ["/bin/bash", "bin/upload.sh"]
        docker-image: taskclusterprivate/upload_symbols:0.0.4
            GECKO_HEAD_REPOSITORY: # see transforms
            GECKO_HEAD_REV: # see transforms
            ARTIFACT_TASKID: {"task-reference": "<build>"}
+   optimization:
+       only-if-dependencies-run: null
        - docker-worker:image:taskclusterprivate/upload_symbols:0.0.4
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -72,16 +72,17 @@ def optimize_task_graph(target_task_grap
 def _make_default_strategies():
     return {
         'never': OptimizationStrategy(),  # "never" is the default behavior
         'index-search': IndexSearch(),
         'seta': SETA(),
         'skip-unless-changed': SkipUnlessChanged(),
         'skip-unless-schedules': SkipUnlessSchedules(),
         'skip-unless-schedules-or-seta': Either(SkipUnlessSchedules(), SETA()),
+        'only-if-dependencies-run': OnlyIfDependenciesRun(),
 def _get_optimizations(target_task_graph, strategies):
     def optimizations(label):
         task = target_task_graph.tasks[label]
         if task.optimization:
             opt_by, arg = task.optimization.items()[0]
@@ -276,16 +277,28 @@ class Either(OptimizationStrategy):
             lambda sub, arg: sub.should_remove_task(task, params, arg))
     def should_replace_task(self, task, params, arg):
         return self._for_substrategies(
             lambda sub, arg: sub.should_replace_task(task, params, arg))
+class OnlyIfDependenciesRun(OptimizationStrategy):
+    """Run this taks only if its dependencies run."""
+    # This takes advantage of the behavior of the second phase of optimization:
+    # a task can only be replaced if it has no un-optimized dependencies. So if
+    # should_replace_task is called, then a task has no un-optimized
+    # dependencies and can be removed (indicated by returning True)
+    def should_replace_task(self, task, params, arg):
+        return True
 class IndexSearch(OptimizationStrategy):
     def should_remove_task(self, task, params, index_paths):
         "If this task has no dependencies, don't run it.."
         return True
     def should_replace_task(self, task, params, index_paths):
         "Look for a task with one of the given index paths"
         for index_path in index_paths:
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -159,16 +159,19 @@ task_description_schema = Schema({
         # consult SETA and skip this task if it is low-value
         {'seta': None},
         # skip this task if none of the given file patterns match
         {'skip-unless-changed': [basestring]},
         # skip this task if unless the change files' SCHEDULES contains any of these components
         {'skip-unless-schedules': list(schedules.ALL_COMPONENTS)},
         # skip if SETA or skip-unless-schedules says to
         {'skip-unless-schedules-or-seta': list(schedules.ALL_COMPONENTS)},
+        # only run this task if its dependencies will run (useful for follow-on tasks that
+        # are unnecessary if the parent tasks are not run)
+        {'only-if-dependencies-run': None}
     # the provisioner-id/worker-type for the task.  The following parameters will
     # be substituted in this string:
     #  {level} -- the scm level of this push
     'worker-type': basestring,
     # Whether the job should use sccache compiler caching.