bug 1423081 - target_tasks_methods updates. r=callek
authorAki Sasaki <asasaki@mozilla.com>
Tue, 05 Dec 2017 19:10:28 -0800
changeset 396350 960aa6859716029636041811e14a3da379d59d83
parent 396349 35d738241395d6f4b182151f12ea71313e904900
child 396351 13d6dfaa9bce4b23bf116064355a8b0babd34a64
push id56975
push userdluca@mozilla.com
push dateThu, 14 Dec 2017 09:59:07 +0000
treeherderautoland@16bcfaad13e1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscallek
bugs1423081
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1423081 - target_tasks_methods updates. r=callek Here we're adding/updating support for the promote/push/ship phases for fennec, devedition, and firefox. These are now keyed off of the new shipping_phase and shipping_product attributes as much as possible. MozReview-Commit-ID: Fkg8jTPeZHZ
taskcluster/taskgraph/target_tasks.py
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -298,98 +298,204 @@ def target_tasks_mozilla_release(full_ta
     """Select the set of tasks required for a promotable beta or release build
     of desktop, plus android CI. The candidates build process involves a pipeline
     of builds and signing, but does not include beetmover or balrog jobs."""
 
     return [l for l, t in full_task_graph.tasks.iteritems() if
             filter_beta_release_tasks(t, parameters)]
 
 
-@_target_task('maple_desktop_promotion')
-@_target_task('mozilla-beta_desktop_promotion')
-@_target_task('mozilla-release_desktop_promotion')
-def target_tasks_mozilla_beta_desktop_promotion(full_task_graph, parameters, graph_config):
+@_target_task('promote_firefox')
+def target_tasks_promote_firefox(full_task_graph, parameters, graph_config):
     """Select the superset of tasks required to promote a beta or release build
-    of desktop. This should include all non-android mozilla_beta tasks, plus
+    of firefox. This should include all non-android mozilla_beta tasks, plus
     l10n, beetmover, balrog, etc."""
 
-    beta_tasks = [l for l, t in full_task_graph.tasks.iteritems() if
-                  filter_beta_release_tasks(t, parameters,
-                                            ignore_kinds=[],
-                                            allow_l10n=True)]
-    allow_kinds = [
-        'build', 'build-signing', 'repackage', 'repackage-signing',
-        'nightly-l10n', 'nightly-l10n-signing', 'repackage-l10n',
-    ]
+    beta_release_tasks = [l for l, t in full_task_graph.tasks.iteritems() if
+                          filter_beta_release_tasks(t, parameters,
+                                                    ignore_kinds=[],
+                                                    allow_l10n=True)]
 
     def filter(task):
         platform = task.attributes.get('build_platform')
 
         # Android has its own promotion.
         if platform and 'android' in platform:
             return False
 
-        if task.kind not in allow_kinds:
+        if platform and 'devedition' in platform:
             return False
 
-        # Allow for beta_tasks; these will get optimized out to point to
-        # the previous graph using ``previous_graph_ids`` and
+        # Allow for {beta,release}_tasks; these will get optimized out to point
+        # to the previous graph using ``previous_graph_ids`` and
         # ``previous_graph_kinds``.
-        if task.label in beta_tasks:
+        # At some point this should filter by shipping_phase == 'build' and
+        # shipping_product matches.
+        if task.label in beta_release_tasks:
             return True
 
+        # 'secondary' update/final verify tasks only run for
+        # RCs
+        if parameters.get('desktop_release_type') != 'rc':
+            if task.kind in ('release-buildbot-update-verify',
+                             'release-update-verify',
+                             'release-secondary-final-verify'):
+                if 'secondary' in task.label:
+                    return False
+
         if task.attributes.get('shipping_product') == 'firefox' and \
                 task.attributes.get('shipping_phase') == 'promote':
             return True
 
-        # TODO: partner repacks
-        # TODO: source task
         # TODO: funsize, all but balrog submission
-        # TODO: bbb update verify
-        # TODO: tc update verify
-        # TODO: beetmover push-to-candidates
-        # TODO: binary transparency
         # TODO: bouncer sub
-        # TODO: snap
         # TODO: recompression tasks
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
-@_target_task('publish_firefox')
-def target_tasks_publish_firefox(full_task_graph, parameters, graph_config):
-    """Select the set of tasks required to publish a candidates build of firefox.
+@_target_task('push_firefox')
+def target_tasks_push_firefox(full_task_graph, parameters, graph_config):
+    """Select the set of tasks required to push a build of firefox to cdns.
     Previous build deps will be optimized out via action task."""
-    filtered_for_candidates = target_tasks_mozilla_beta_desktop_promotion(
+    filtered_for_candidates = target_tasks_promote_firefox(
+        full_task_graph, parameters, graph_config,
+    )
+
+    def filter(task):
+        # Include promotion tasks; these will be optimized out
+        if task.label in filtered_for_candidates:
+            return True
+        if task.attributes.get('shipping_product') == 'firefox' and \
+                task.attributes.get('shipping_phase') == 'push':
+            return True
+        # TODO: add beetmover push-to-releases
+        # TODO: publish to balrog
+        # TODO: funsize balrog submission
+        # TODO: recompression push-to-releases + balrog
+        # TODO: checksums
+
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+
+@_target_task('ship_firefox')
+def target_tasks_ship_firefox(full_task_graph, parameters, graph_config):
+    """Select the set of tasks required to ship firefox.
+    Previous build deps will be optimized out via action task."""
+    filtered_for_candidates = target_tasks_push_firefox(
         full_task_graph, parameters, graph_config,
     )
 
     def filter(task):
         # Include promotion tasks; these will be optimized out
         if task.label in filtered_for_candidates:
             return True
         if task.attributes.get('shipping_product') == 'firefox' and \
-                task.attributes.get('shipping_phase') in ('publish', 'ship'):
+                task.attributes.get('shipping_phase') == 'ship':
             return True
         # TODO: add beetmover push-to-releases
-        # TODO: tagging / version bumping
         # TODO: publish to balrog
         # TODO: funsize balrog submission
         # TODO: recompression push-to-releases + balrog
-        # TODO: final verify
-        # TODO: uptake monitoring
-        # TODO: bouncer aliases
         # TODO: checksums
-        # TODO: shipit mark as shipped
+
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+
+@_target_task('promote_devedition')
+def target_tasks_promote_devedition(full_task_graph, parameters, graph_config):
+    """Select the superset of tasks required to promote a beta or release build
+    of devedition. This should include all non-android mozilla_beta tasks, plus
+    l10n, beetmover, balrog, etc."""
+
+    beta_release_tasks = [l for l, t in full_task_graph.tasks.iteritems() if
+                          filter_beta_release_tasks(t, parameters,
+                                                    ignore_kinds=[],
+                                                    allow_l10n=True)]
+
+    def filter(task):
+        platform = task.attributes.get('build_platform')
+
+        # Android has its own promotion.
+        if platform and 'android' in platform:
+            return False
 
-    return [l for l, t in full_task_graph.iteritems() if filter(t)]
+        if platform and 'devedition' not in platform:
+            return False
+
+        # Allow for {beta,release}_tasks; these will get optimized out to point to
+        # the previous graph using ``previous_graph_ids`` and
+        # ``previous_graph_kinds``.
+        # At some point this should filter by shipping_phase == 'build' and
+        # shipping_product matches.
+        if task.label in beta_release_tasks:
+            return True
+
+        if task.attributes.get('shipping_product') == 'devedition' and \
+                task.attributes.get('shipping_phase') == 'promote':
+            return True
+
+        # TODO: funsize, all but balrog submission
+        # TODO: binary transparency
+        # TODO: bouncer sub
+        # TODO: recompression tasks
+
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
-@_target_task('candidates_fennec')
-def target_tasks_candidates_fennec(full_task_graph, parameters, graph_config):
+@_target_task('push_devedition')
+def target_tasks_push_devedition(full_task_graph, parameters, graph_config):
+    """Select the set of tasks required to push a build of devedition to cdns.
+    Previous build deps will be optimized out via action task."""
+    filtered_for_candidates = target_tasks_promote_devedition(
+        full_task_graph, parameters, graph_config,
+    )
+
+    def filter(task):
+        # Include promotion tasks; these will be optimized out
+        if task.label in filtered_for_candidates:
+            return True
+        if task.attributes.get('shipping_product') == 'devedition' and \
+                task.attributes.get('shipping_phase') == 'push':
+            return True
+        # TODO: add beetmover push-to-releases
+        # TODO: publish to balrog
+        # TODO: funsize balrog submission
+        # TODO: recompression push-to-releases + balrog
+        # TODO: checksums
+
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+
+@_target_task('ship_devedition')
+def target_tasks_ship_devedition(full_task_graph, parameters, graph_config):
+    """Select the set of tasks required to ship devedition.
+    Previous build deps will be optimized out via action task."""
+    filtered_for_candidates = target_tasks_push_devedition(
+        full_task_graph, parameters, graph_config,
+    )
+
+    def filter(task):
+        # Include promotion tasks; these will be optimized out
+        if task.label in filtered_for_candidates:
+            return True
+        if task.attributes.get('shipping_product') == 'devedition' and \
+                task.attributes.get('shipping_phase') == 'ship':
+            return True
+        # TODO: add beetmover push-to-releases
+        # TODO: publish to balrog
+        # TODO: funsize balrog submission
+        # TODO: recompression push-to-releases + balrog
+        # TODO: checksums
+
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+
+@_target_task('promote_fennec')
+def target_tasks_promote_fennec(full_task_graph, parameters, graph_config):
     """Select the set of tasks required for a candidates build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
     filtered_for_project = target_tasks_nightly_fennec(full_task_graph, parameters, graph_config)
 
     def filter(task):
         attr = task.attributes.get
         # Don't ship single locale fennec anymore - Bug 1408083
@@ -401,30 +507,30 @@ def target_tasks_candidates_fennec(full_
                     return True
         if task.attributes.get('shipping_product') == 'fennec' and \
                 task.attributes.get('shipping_phase') == 'promote':
             return True
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(full_task_graph[l])]
 
 
-@_target_task('publish_fennec')
-def target_tasks_publish_fennec(full_task_graph, parameters, graph_config):
-    """Select the set of tasks required to publish a candidates build of fennec.
+@_target_task('ship_fennec')
+def target_tasks_ship_fennec(full_task_graph, parameters, graph_config):
+    """Select the set of tasks required to ship fennec.
     Previous build deps will be optimized out via action task."""
-    filtered_for_candidates = target_tasks_candidates_fennec(
+    filtered_for_candidates = target_tasks_promote_fennec(
         full_task_graph, parameters, graph_config,
     )
 
     def filter(task):
         # Include candidates build tasks; these will be optimized out
         if task.label in filtered_for_candidates:
             return True
         if task.attributes.get('shipping_product') == 'fennec' and \
-                task.attributes.get('shipping_phase') in ('ship', 'publish'):
+                task.attributes.get('shipping_phase') in ('ship', 'push'):
             return True
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(full_task_graph[l])]
 
 
 @_target_task('pine_tasks')
 def target_tasks_pine(full_task_graph, parameters, graph_config):
     """Bug 1339179 - no mobile automation needed on pine"""