Bug 1526419 - add mar-signing-autograph-stage task r=Callek
authorAki Sasaki <asasaki@mozilla.com>
Fri, 01 Mar 2019 23:53:24 +0000
changeset 519936 811caa480654d82a931322faa2a9b38c766212c9
parent 519935 942dd126731278108778aba254469079e3697f9f
child 519937 8ffa5f870efb9ae97f73072bafb3a870b3cbec73
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek
bugs1526419
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1526419 - add mar-signing-autograph-stage task r=Callek We use autograph-prod for our ci, nightly, and release signing. Autograph-stage doesn't have the same guarantees for availability, so pointing, say, dep-signing at autograph-stage would have resulted in occasional tree closures whenever autograph-stage changes configuration or is down. However, we also want a way to verify autograph-stage is still valid, after the autograph team makes changes. This task is meant to be add-task'ed; a green result means autograph-stage has signed the mar file correctly. Differential Revision: https://phabricator.services.mozilla.com/D20749
taskcluster/ci/config.yml
taskcluster/ci/mar-signing-autograph-stage/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/mar_signing.py
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -80,16 +80,17 @@ treeherder:
         'WMC32': 'MinGW-Clang builds for Windows 32-bits'
         'WMC64': 'MinGW-Clang builds for Windows 64-bits'
         'Searchfox': 'Searchfox builds'
         'SM': 'Spidermonkey builds'
         'pub': 'APK publishing'
         'p': 'Partial generation'
         'ps': 'Partials signing'
         'ms': 'Complete MAR signing'
+        'ms-stage': 'Autograph-stage MAR signing test'
         'Rel': 'Release promotion'
         'Snap': 'Snap image generation'
         'langpack': 'Langpack sigatures and uploads'
         'TPS': 'Sync tests'
         'UV': 'Update verify'
         'pipfu': 'pipfile update'
         'WR': 'WebRender standalone'
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/mar-signing-autograph-stage/kind.yml
@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+    - taskgraph.transforms.name_sanity:transforms
+    - taskgraph.transforms.mar_signing:transforms
+    - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+    - repackage
+
+only-for-build-platforms:
+    - linux64-nightly/opt
+
+job-template:
+    shipping-phase: null
+    treeherder-group: ms-stage
+    treeherder:
+        tier: 3
+    description-suffix: 'autograph-stage mar signing test'
+    required_signoffs:
+        - mar-signing
+    run-on-projects: []
+    nightly: false
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -457,16 +457,20 @@ Repackage-signing-l10n take the repackag
 mar-signing
 -----------
 Mar-signing takes the complete update MARs and signs them.
 
 mar-signing-l10n
 ----------------
 Mar-signing-l10n takes the complete update MARs and signs them for localized versions.
 
+mar-signing-autograph-stage
+---------------------------
+These tasks are only to test autograph-stage, when the autograph team asks for their staging environment to be tested.
+
 repackage-msi
 -------------
 Repackage-msi takes the signed full installer and produces an msi installer (that wraps the full installer)
 Using the ```./mach repackage``` command
 
 repackage-signing-msi
 ---------------------
 Repackage-signing-msi takes the repackaged msi installers and signs them.
--- a/taskcluster/taskgraph/transforms/mar_signing.py
+++ b/taskcluster/taskgraph/transforms/mar_signing.py
@@ -18,18 +18,23 @@ from taskgraph.util.scriptworker import 
 from taskgraph.util.partials import get_balrog_platform_name, get_partials_artifacts
 from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.util.treeherder import join_symbol
 
 import logging
 logger = logging.getLogger(__name__)
 
 SIGNING_FORMATS = {
-    'target.complete.mar': ['autograph_hash_only_mar384'],
-    'target.bz2.complete.mar': ['mar'],
+    'mar-signing-autograph-stage': {
+        'target.complete.mar': ['autograph_stage_mar384'],
+    },
+    'default': {
+        'target.complete.mar': ['autograph_hash_only_mar384'],
+        'target.bz2.complete.mar': ['mar'],
+    },
 }
 
 transforms = TransformSequence()
 
 
 def generate_partials_artifacts(job, release_history, platform, locale=None):
     artifact_prefix = get_artifact_prefix(job)
     if locale:
@@ -66,45 +71,44 @@ def generate_partials_artifacts(job, rel
     }
 
     if old_mar_upstream_artifacts["paths"]:
         upstream_artifacts.append(old_mar_upstream_artifacts)
 
     return upstream_artifacts
 
 
-def generate_complete_artifacts(job):
+def generate_complete_artifacts(job, kind):
     upstream_artifacts = []
+    if kind not in SIGNING_FORMATS:
+        kind = 'default'
     for artifact in job.release_artifacts:
         basename = os.path.basename(artifact)
-        if basename in SIGNING_FORMATS:
+        if basename in SIGNING_FORMATS[kind]:
             upstream_artifacts.append({
                 "taskId": {"task-reference": '<{}>'.format(job.kind)},
                 "taskType": 'build',
                 "paths": [artifact],
-                "formats": SIGNING_FORMATS[basename],
+                "formats": SIGNING_FORMATS[kind][basename],
             })
 
     return upstream_artifacts
 
 
 @transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
         dep_job = job['primary-dependency']
         locale = dep_job.attributes.get('locale')
 
         treeherder = job.get('treeherder', {})
-        treeherder['symbol'] = join_symbol(
-            job.get('treeherder-group', 'ms'),
-            locale or 'N'
+        treeherder.setdefault(
+            'symbol', join_symbol(job.get('treeherder-group', 'ms'), locale or 'N')
         )
 
-        dep_th_platform = dep_job.task.get('extra', {}).get(
-            'treeherder', {}).get('machine', {}).get('platform', '')
         label = job.get('label', "{}-{}".format(config.kind, dep_job.label))
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
         treeherder.setdefault('platform',
                               "{}/opt".format(dep_th_platform))
         treeherder.setdefault('kind', 'build')
         treeherder.setdefault('tier', 1)
 
@@ -123,20 +127,20 @@ def make_task_description(config, jobs):
         if locale:
             attributes['locale'] = locale
 
         balrog_platform = get_balrog_platform_name(dep_th_platform)
         if config.kind == 'partials-signing':
             upstream_artifacts = generate_partials_artifacts(
                 dep_job, config.params['release_history'], balrog_platform, locale)
         else:
-            upstream_artifacts = generate_complete_artifacts(dep_job)
+            upstream_artifacts = generate_complete_artifacts(dep_job, config.kind)
 
         build_platform = dep_job.attributes.get('build_platform')
-        is_nightly = dep_job.attributes.get('nightly')
+        is_nightly = job.get('nightly', dep_job.attributes.get('nightly'))
         signing_cert_scope = get_signing_cert_scope_per_platform(
             build_platform, is_nightly, config
         )
 
         scopes = [signing_cert_scope] + list({
             add_scope_prefix(config, 'signing:format:{}'.format(format))
             for artifact in upstream_artifacts
             for format in artifact['formats']
@@ -148,13 +152,14 @@ def make_task_description(config, jobs):
                 dep_job.task["metadata"]["description"], job['description-suffix']),
             'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': upstream_artifacts,
                        'max-run-time': 3600},
             'dependencies': dependencies,
             'attributes': attributes,
             'scopes': scopes,
-            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'run-on-projects': job.get('run-on-projects',
+                                       dep_job.attributes.get('run_on_projects')),
             'treeherder': treeherder,
         }
 
         yield task