Bug 1383880: add only-if-dependencies-run optimization for follow-ons; r=ahal
☠☠ backed out by d88e5dec2638 ☠ ☠
authorDustin J. Mitchell <dustin@mozilla.com>
Fri, 25 Aug 2017 21:18:13 +0000
changeset 431653 ec2b8ba5a949c3b899a647dbad93aa60b653e745
parent 431652 8e5847d9acda6349226b37fbbfc79ef2c01d7614
child 431654 571a6c9054a540e0bf40baf837da9175f21796b9
push id7785
push userryanvm@gmail.com
push dateThu, 21 Sep 2017 13:39:55 +0000
treeherdermozilla-beta@06d4034a8a03 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1383880
milestone57.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1383880: add only-if-dependencies-run optimization for follow-ons; r=ahal MozReview-Commit-ID: JuhwyQIx3Mh
taskcluster/ci/upload-generated-sources/kind.yml
taskcluster/ci/upload-symbols/kind.yml
taskcluster/taskgraph/optimize.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/upload-generated-sources/kind.yml
+++ b/taskcluster/ci/upload-generated-sources/kind.yml
@@ -26,10 +26,12 @@ job-template:
   worker:
      docker-image: {in-tree: "lint"}
      max-run-time: 600
   run:
     using: run-task
     command: >
             cd /builds/worker/checkouts/gecko &&
             ./mach python build/upload_generated_sources.py ${ARTIFACT_URL}
+  optimization:
+    only-if-dependencies-run: null
   scopes:
       - secrets:get:project/releng/gecko/build/level-{level}/gecko-generated-sources-upload
--- a/taskcluster/ci/upload-symbols/kind.yml
+++ b/taskcluster/ci/upload-symbols/kind.yml
@@ -38,10 +38,12 @@ job-template:
        os: linux
        max-run-time: 600
        command: ["/bin/bash", "bin/upload.sh"]
        docker-image: taskclusterprivate/upload_symbols:0.0.4
        env:
            GECKO_HEAD_REPOSITORY: # see transforms
            GECKO_HEAD_REV: # see transforms
            ARTIFACT_TASKID: {"task-reference": "<build>"}
+   optimization:
+       only-if-dependencies-run: null
    scopes:
        - docker-worker:image:taskclusterprivate/upload_symbols:0.0.4
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -72,16 +72,17 @@ def optimize_task_graph(target_task_grap
 def _make_default_strategies():
     return {
         'never': OptimizationStrategy(),  # "never" is the default behavior
         'index-search': IndexSearch(),
         'seta': SETA(),
         'skip-unless-changed': SkipUnlessChanged(),
         'skip-unless-schedules': SkipUnlessSchedules(),
         'skip-unless-schedules-or-seta': Either(SkipUnlessSchedules(), SETA()),
+        'only-if-dependencies-run': OnlyIfDependenciesRun(),
     }
 
 
 def _get_optimizations(target_task_graph, strategies):
     def optimizations(label):
         task = target_task_graph.tasks[label]
         if task.optimization:
             opt_by, arg = task.optimization.items()[0]
@@ -276,16 +277,28 @@ class Either(OptimizationStrategy):
             lambda sub, arg: sub.should_remove_task(task, params, arg))
 
     def should_replace_task(self, task, params, arg):
         return self._for_substrategies(
             arg,
             lambda sub, arg: sub.should_replace_task(task, params, arg))
 
 
+class OnlyIfDependenciesRun(OptimizationStrategy):
+    """Run this taks only if its dependencies run."""
+
+    # This takes advantage of the behavior of the second phase of optimization:
+    # a task can only be replaced if it has no un-optimized dependencies. So if
+    # should_replace_task is called, then a task has no un-optimized
+    # dependencies and can be removed (indicated by returning True)
+
+    def should_replace_task(self, task, params, arg):
+        return True
+
+
 class IndexSearch(OptimizationStrategy):
     def should_remove_task(self, task, params, index_paths):
         "If this task has no dependencies, don't run it.."
         return True
 
     def should_replace_task(self, task, params, index_paths):
         "Look for a task with one of the given index paths"
         for index_path in index_paths:
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -179,16 +179,19 @@ task_description_schema = Schema({
         # consult SETA and skip this task if it is low-value
         {'seta': None},
         # skip this task if none of the given file patterns match
         {'skip-unless-changed': [basestring]},
         # skip this task if unless the change files' SCHEDULES contains any of these components
         {'skip-unless-schedules': list(schedules.ALL_COMPONENTS)},
         # skip if SETA or skip-unless-schedules says to
         {'skip-unless-schedules-or-seta': list(schedules.ALL_COMPONENTS)},
+        # only run this task if its dependencies will run (useful for follow-on tasks that
+        # are unnecessary if the parent tasks are not run)
+        {'only-if-dependencies-run': None}
     ),
 
     # the provisioner-id/worker-type for the task.  The following parameters will
     # be substituted in this string:
     #  {level} -- the scm level of this push
     'worker-type': basestring,
 
     # Whether the job should use sccache compiler caching.