Bug 1381577 - Part Z; Slight refactor to beetmover-repackage to make windows additions clearer. r=kmoir
authorJustin Wood <Callek@gmail.com>
Tue, 18 Jul 2017 16:05:33 -0400
changeset 418989 d1c0752fff0499ca1e78efcb8698ab5e8c05df83
parent 418988 152bc4406fa9623a008ab1b052c82dd89aa14099
child 418990 176d94a4c3d7db415531b0bfd717f9736abed932
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerskmoir
bugs1381577
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1381577 - Part Z; Slight refactor to beetmover-repackage to make windows additions clearer. r=kmoir Land date changes to support windows nightlies onto central Supports beetmoving from the build-signing task, as well as ignoring dependencies if we have nothing to beetmove in them (e.g. build-signing for OSX) MozReview-Commit-ID: 24byn1posKT
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -35,16 +35,17 @@ from voluptuous import Any, Required, Op
     "target.txt",
     "target.web-platform.tests.tar.gz",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "target.jsshell.zip",
     "mozharness.zip",
     "target.langpack.xpi",
 ]
+
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
     "update/target.complete.mar",
 ]
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -37,16 +37,17 @@ logger = logging.getLogger(__name__)
     "target.txt",
     "target.web-platform.tests.tar.gz",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "target.jsshell.zip",
     "mozharness.zip",
     "target.langpack.xpi",
 ]
+
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
     "target.langpack.xpi",
     "balrog_props.json",
 ]
@@ -61,16 +62,22 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
         "host/bin/mbsdiff",
     ],
     'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
 }
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
+UPSTREAM_ARTIFACT_SIGNED_PATHS = {
+}
+# Until bug 1331141 is fixed, if you are adding any new artifacts here that
+# need to be transfered to S3, please be aware you also need to follow-up
+# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
+# See example in bug 1348286
 UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
     'macosx64-nightly': ['target.dmg'],
     'macosx64-nightly-l10n': ['target.dmg'],
 }
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
@@ -132,23 +139,24 @@ def make_task_description(config, jobs):
                               "{}/opt".format(dep_th_platform))
         treeherder.setdefault('tier', 1)
         treeherder.setdefault('kind', 'build')
         label = job.get('label', "beetmover-{}".format(dep_job.label))
 
         dependent_kind = str(dep_job.kind)
         dependencies = {dependent_kind: dep_job.label}
 
-        # macosx nightly builds depend on repackage which use in tree docker
-        # images and thus have two dependencies
-        # change the signing_dependencies to be use the ones in
-        docker_dependencies = {"docker-image":
-                               dep_job.dependencies['docker-image']
-                               }
-        dependencies.update(docker_dependencies)
+        if 'docker-image' in dep_job.dependencies:
+            # macosx nightly builds depend on repackage which use in tree
+            # docker images and thus have two dependencies
+            # change the signing_dependencies to be use the ones in
+            docker_dependencies = {"docker-image":
+                                   dep_job.dependencies['docker-image']
+                                   }
+            dependencies.update(docker_dependencies)
 
         signing_name = "build-signing"
         if job.get('locale'):
             signing_name = "nightly-l10n-signing"
         signing_dependencies = {signing_name:
                                 dep_job.dependencies[signing_name]
                                 }
         dependencies.update(signing_dependencies)
@@ -185,80 +193,94 @@ def make_task_description(config, jobs):
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
         }
 
         yield task
 
 
-def generate_upstream_artifacts(build_task_ref, repackage_task_ref,
-                                repackage_signing_task_ref, platform,
-                                locale=None):
+def generate_upstream_artifacts(build_task_ref, build_signing_task_ref,
+                                repackage_task_ref, repackage_signing_task_ref,
+                                platform, locale=None):
 
     build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
+    build_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
     repackage_mapping = UPSTREAM_ARTIFACT_REPACKAGE_PATHS
     repackage_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS
 
     artifact_prefix = 'public/build'
     if locale:
         artifact_prefix = 'public/build/{}'.format(locale)
         platform = "{}-l10n".format(platform)
 
-    upstream_artifacts = [{
-        "taskId": {"task-reference": build_task_ref},
-        "taskType": "build",
-        "paths": ["{}/{}".format(artifact_prefix, p)
-                  for p in build_mapping[platform]],
-        "locale": locale or "en-US",
-        }, {
-        "taskId": {"task-reference": repackage_task_ref},
-        "taskType": "repackage",
-        "paths": ["{}/{}".format(artifact_prefix, p)
-                  for p in repackage_mapping[platform]],
-        "locale": locale or "en-US",
-        }, {
-        "taskId": {"task-reference": repackage_signing_task_ref},
-        "taskType": "repackage",
-        "paths": ["{}/{}".format(artifact_prefix, p)
-                  for p in repackage_signing_mapping[platform]],
-        "locale": locale or "en-US",
-    }]
+    upstream_artifacts = []
+
+    task_refs = [
+        build_task_ref,
+        build_signing_task_ref,
+        repackage_task_ref,
+        repackage_signing_task_ref
+    ]
+    tasktypes = ['build', 'signing', 'repackage', 'repackage']
+    mapping = [
+        build_mapping,
+        build_signing_mapping,
+        repackage_mapping,
+        repackage_signing_mapping
+    ]
+
+    for ref, tasktype, mapping in zip(task_refs, tasktypes, mapping):
+        if platform in mapping:
+            upstream_artifacts.append({
+                "taskId": {"task-reference": ref},
+                "taskType": tasktype,
+                "paths": ["{}/{}".format(artifact_prefix, p)
+                          for p in mapping[platform]],
+                "locale": locale or "en-US",
+            })
 
     return upstream_artifacts
 
 
+def is_valid_beetmover_job(job):
+    # windows builds don't have docker-image, so fewer dependencies
+    return (len(job["dependencies"]) == 5 and
+            any(['repackage' in j for j in job['dependencies']]))
+
+
 @transforms.add
 def make_task_worker(config, jobs):
     for job in jobs:
-        valid_beetmover_job = (len(job["dependencies"]) == 5 and
-                               any(['repackage' in j for j in job['dependencies']]))
-        if not valid_beetmover_job:
+        if not is_valid_beetmover_job(job):
             raise NotImplementedError("Beetmover_repackage must have five dependencies.")
 
         locale = job["attributes"].get("locale")
         platform = job["attributes"]["build_platform"]
         build_task = None
+        build_signing_task = None
         repackage_task = None
         repackage_signing_task = None
         for dependency in job["dependencies"].keys():
             if 'repackage-signing' in dependency:
                 repackage_signing_task = dependency
             elif 'repackage' in dependency:
                 repackage_task = dependency
             elif 'signing' in dependency:
-                pass
+                # catches build-signing and nightly-l10n-signing
+                build_signing_task = dependency
             else:
                 build_task = "build"
 
         build_task_ref = "<" + str(build_task) + ">"
+        build_signing_task_ref = "<" + str(build_signing_task) + ">"
         repackage_task_ref = "<" + str(repackage_task) + ">"
         repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">"
         upstream_artifacts = generate_upstream_artifacts(
-            build_task_ref, repackage_task_ref,
+            build_task_ref, build_signing_task_ref, repackage_task_ref,
             repackage_signing_task_ref, platform, locale
         )
 
         worker = {'implementation': 'beetmover',
                   'upstream-artifacts': upstream_artifacts}
         if locale:
             worker["locale"] = locale
         job["worker"] = worker