Bug 1457010 - gpg sign partner repacks, r=aki
authorNick Thomas <nthomas@mozilla.com>
Thu, 26 Apr 2018 10:52:03 +1200
changeset 469233 b59f651f46ec54a14be43db5daeb4618495b7b4e
parent 469232 122be576bdc1c03e9c8fce0a709b74018873eb2c
child 469234 ab1472823e8a2932c504aec75081f9bbf5276246
push id9165
push userasasaki@mozilla.com
push dateThu, 26 Apr 2018 21:04:54 +0000
treeherdermozilla-beta@064c3804de2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1457010
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1457010 - gpg sign partner repacks, r=aki This adds repackage-signing on mac and linux, depending on repackage and the chunking-dummy kinds respectively, and repackage-signing is extended to create gpg signatures. The signing_dependencies are no longer added because the beetmover_repackage_partner.py transform is going to set that up manually, and it avoids duplicate targets which the schema blocks. Beetmover can depend now on repackage-signing for all platforms, and no longer has any indirect dependencies to worry about, but does need to know about copying the .asc files as upstream artifacts. MozReview-Commit-ID: JcIdXQ2B7Rg
taskcluster/ci/release-eme-free-repack-beetmover/kind.yml
taskcluster/ci/release-eme-free-repack-repackage-signing/kind.yml
taskcluster/ci/release-partner-repack-beetmover/kind.yml
taskcluster/ci/release-partner-repack-repackage-signing/kind.yml
taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
taskcluster/taskgraph/transforms/repackage_signing_partner.py
--- a/taskcluster/ci/release-eme-free-repack-beetmover/kind.yml
+++ b/taskcluster/ci/release-eme-free-repack-beetmover/kind.yml
@@ -6,18 +6,17 @@ loader: taskgraph.loader.single_dep:load
 
 transforms:
    - taskgraph.transforms.name_sanity:transforms
    - taskgraph.transforms.beetmover_repackage_partner:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-   - release-eme-free-repack-repackage  # Mac
-   - release-eme-free-repack-repackage-signing  # Windows
+   - release-eme-free-repack-repackage-signing
 
 only-for-build-platforms:
    - macosx64-nightly/opt
    - win32-nightly/opt
    - win64-nightly/opt
 
 job-template:
    shipping-phase: promote
--- a/taskcluster/ci/release-eme-free-repack-repackage-signing/kind.yml
+++ b/taskcluster/ci/release-eme-free-repack-repackage-signing/kind.yml
@@ -11,8 +11,9 @@ transforms:
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - release-eme-free-repack-repackage
 
 only-for-build-platforms:
    - win32-nightly/opt
    - win64-nightly/opt
+   - macosx64-nightly/opt
--- a/taskcluster/ci/release-partner-repack-beetmover/kind.yml
+++ b/taskcluster/ci/release-partner-repack-beetmover/kind.yml
@@ -6,19 +6,17 @@ loader: taskgraph.loader.single_dep:load
 
 transforms:
    - taskgraph.transforms.name_sanity:transforms
    - taskgraph.transforms.beetmover_repackage_partner:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-   - release-partner-repack-chunking-dummy  # Linux
-   - release-partner-repack-repackage  # Mac
-   - release-partner-repack-repackage-signing  # Windows
+   - release-partner-repack-repackage-signing
 
 only-for-build-platforms:
    - linux-nightly/opt
    - linux64-nightly/opt
    - macosx64-nightly/opt
    - win32-nightly/opt
    - win64-nightly/opt
 
--- a/taskcluster/ci/release-partner-repack-repackage-signing/kind.yml
+++ b/taskcluster/ci/release-partner-repack-repackage-signing/kind.yml
@@ -6,13 +6,17 @@ loader: taskgraph.loader.single_dep:load
 
 transforms:
    - taskgraph.transforms.name_sanity:transforms
    - taskgraph.transforms.repackage_signing_partner:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-   - release-partner-repack-repackage
+   - release-partner-repack-chunking-dummy  # Linux
+   - release-partner-repack-repackage  # Windows, Mac
 
 only-for-build-platforms:
+   - linux-nightly/opt
+   - linux64-nightly/opt
+   - macosx64-nightly/opt
    - win32-nightly/opt
    - win64-nightly/opt
--- a/taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
@@ -75,43 +75,16 @@ def validate(config, jobs):
         label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
         validate_schema(
             beetmover_description_schema, job,
             "In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
         yield job
 
 
 @transforms.add
-def skip_for_indirect_dependencies(config, jobs):
-    for job in jobs:
-        dep_job = job['dependent-task']
-        build_platform = dep_job.attributes.get("build_platform")
-        if not build_platform:
-            raise Exception("Cannot find build platform!")
-
-        # Partner and EME free beetmover tasks have multiple upstreams defined
-        # because some platforms don't run some parts of the sign -> repack ->
-        # repack sign chain. We only want to run beetmover for the last part of
-        # that chain that runs for any given platform.
-        # For Linux, it is the eme-free/partner repack build tasks.
-        # For Mac, it is repackage.
-        # For Windows, it is repackage-signing.
-        if "win" in build_platform:
-            if "repackage" not in dep_job.label:
-                continue
-            elif "signing" not in dep_job.label:
-                continue
-        if "macosx" in build_platform:
-            if "repackage" not in dep_job.label:
-                continue
-
-        yield job
-
-
-@transforms.add
 def resolve_keys(config, jobs):
     for job in jobs:
         resolve_keyed_by(
             job, 'partner-bucket-scope', item_name=job['label'], project=config.params['project']
         )
         yield job
 
 
@@ -145,20 +118,19 @@ def make_task_description(config, jobs):
         base_label = "release-partner-repack"
         if "eme" in config.kind:
             base_label = "release-eme-free-repack"
         dependencies["build"] = "{}-{}".format(base_label, build_platform)
         if "macosx" in build_platform or "win" in build_platform:
             dependencies["repackage"] = "{}-repackage-{}-{}".format(
                 base_label, build_platform, repack_id.replace('/', '-')
             )
-        if "win" in build_platform:
-            dependencies["repackage-signing"] = "{}-repackage-signing-{}-{}".format(
-                base_label, build_platform, repack_id.replace('/', '-')
-            )
+        dependencies["repackage-signing"] = "{}-repackage-signing-{}-{}".format(
+             base_label, build_platform, repack_id.replace('/', '-')
+        )
 
         attributes = copy_attributes_from_dependent_job(dep_job)
 
         task = {
             'label': label,
             'description': description,
             'dependencies': dependencies,
             'attributes': attributes,
@@ -217,30 +189,48 @@ def generate_upstream_artifacts(job, bui
 
     if "linux" in platform:
         upstream_artifacts.append({
             "taskId": {"task-reference": build_task_ref},
             "taskType": "build",
             "paths": ["{}/{}/target.tar.bz2".format(artifact_prefix, repack_id)],
             "locale": partner_path,
         })
+        upstream_artifacts.append({
+            "taskId": {"task-reference": repackage_signing_task_ref},
+            "taskType": "repackage",
+            "paths": ["{}/{}/target.tar.bz2.asc".format(artifact_prefix, repack_id)],
+            "locale": partner_path,
+        })
     elif "macosx" in platform:
         upstream_artifacts.append({
             "taskId": {"task-reference": repackage_task_ref},
             "taskType": "repackage",
             "paths": ["{}/{}/target.dmg".format(artifact_prefix, repack_id)],
             "locale": partner_path,
         })
+        upstream_artifacts.append({
+            "taskId": {"task-reference": repackage_signing_task_ref},
+            "taskType": "repackage",
+            "paths": ["{}/{}/target.dmg.asc".format(artifact_prefix, repack_id)],
+            "locale": partner_path,
+        })
     elif "win" in platform:
         upstream_artifacts.append({
             "taskId": {"task-reference": repackage_signing_task_ref},
             "taskType": "repackage",
             "paths": ["{}/{}/target.installer.exe".format(artifact_prefix, repack_id)],
             "locale": partner_path,
         })
+        upstream_artifacts.append({
+            "taskId": {"task-reference": repackage_signing_task_ref},
+            "taskType": "repackage",
+            "paths": ["{}/{}/target.installer.exe.asc".format(artifact_prefix, repack_id)],
+            "locale": partner_path,
+        })
 
     if not upstream_artifacts:
         raise Exception("Couldn't find any upstream artifacts.")
 
     return upstream_artifacts
 
 
 @transforms.add
--- a/taskcluster/taskgraph/transforms/repackage_signing_partner.py
+++ b/taskcluster/taskgraph/transforms/repackage_signing_partner.py
@@ -48,56 +48,78 @@ def validate(config, jobs):
 
 
 @transforms.add
 def make_repackage_signing_description(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
         repack_id = dep_job.task['extra']['repack_id']
         attributes = dep_job.attributes
+        build_platform = dep_job.attributes.get('build_platform')
+        is_nightly = dep_job.attributes.get('nightly')
 
+        # Mac & windows
         label = dep_job.label.replace("repackage-", "repackage-signing-")
+        # Linux
+        label = label.replace("chunking-dummy-", "repackage-signing-")
         description = (
             "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '"
             "{build_platform}/{build_type}'".format(
                 repack_id=repack_id,
                 build_platform=attributes.get('build_platform'),
                 build_type=attributes.get('build_type')
             )
         )
 
-        dependencies = {"repackage": dep_job.label}
+        if 'linux' in build_platform:
+            # we want the repack job, via the dependencies for the the chunking-dummy dep_job
+            for dep in dep_job.dependencies.values():
+                if dep.startswith('release-partner-repack'):
+                    dependencies = {"repack": dep}
+                    break
+        else:
+            # we have a genuine repackage job as our parent
+            dependencies = {"repackage": dep_job.label}
 
-        signing_dependencies = dep_job.dependencies
-        # This is so we get the build task etc in our dependencies to
-        # have better beetmover support.
-        dependencies.update({k: v for k, v in signing_dependencies.items()
-                             if k != 'docker-image'})
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes['repackage_type'] = 'repackage-signing'
 
-        build_platform = dep_job.attributes.get('build_platform')
-        is_nightly = dep_job.attributes.get('nightly')
         signing_cert_scope = get_signing_cert_scope_per_platform(
             build_platform, is_nightly, config
         )
-        scopes = [signing_cert_scope]
-
-        if 'win' not in build_platform:
-            raise Exception("Repackage signing is not supported for non-Windows partner repacks.")
+        scopes = [signing_cert_scope, add_scope_prefix(config, 'signing:format:gpg')]
 
-        upstream_artifacts = [{
-            "taskId": {"task-reference": "<repackage>"},
-            "taskType": "repackage",
-            "paths": [
-                get_artifact_path(dep_job, "{}/target.installer.exe".format(repack_id)),
-            ],
-            "formats": ["sha2signcode"]
-        }]
-        scopes.append(add_scope_prefix(config, "signing:format:sha2signcode"))
+        if 'win' in build_platform:
+            upstream_artifacts = [{
+                "taskId": {"task-reference": "<repackage>"},
+                "taskType": "repackage",
+                "paths": [
+                    get_artifact_path(dep_job, "{}/target.installer.exe".format(repack_id)),
+                ],
+                "formats": ["sha2signcode", "gpg"]
+            }]
+            scopes.append(add_scope_prefix(config, "signing:format:sha2signcode"))
+        elif 'mac' in build_platform:
+            upstream_artifacts = [{
+                "taskId": {"task-reference": "<repackage>"},
+                "taskType": "repackage",
+                "paths": [
+                    get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)),
+                ],
+                "formats": ["gpg"]
+            }]
+        elif 'linux' in build_platform:
+            upstream_artifacts = [{
+                "taskId": {"task-reference": "<repack>"},
+                "taskType": "repackage",
+                "paths": [
+                    get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)),
+                ],
+                "formats": ["gpg"]
+            }]
 
         task = {
             'label': label,
             'description': description,
             # 'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,