Bug 1442545: [partner-repack] Add repack partner builds; r=Callek, a=release
authorTom Prince <mozilla@hocat.ca>
Wed, 18 Apr 2018 11:02:02 -0600
changeset 463421 79fb7fc8097812c884b79998702af408ca682f13
parent 463420 3b9072ca98e5c623259d7dc8c2c42d2ed3b0399b
child 463422 6ac25d14537e695acfa8e3640de02049df578c70
push id1683
push usersfraser@mozilla.com
push dateThu, 26 Apr 2018 16:43:40 +0000
treeherdermozilla-release@5af6cb21869d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek, release
bugs1442545
milestone60.0
Bug 1442545: [partner-repack] Add repack partner builds; r=Callek, a=release Differential Revision: https://phabricator.services.mozilla.com/D980
taskcluster/ci/release-eme-free-repack/kind.yml
taskcluster/ci/release-partner-repack/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/loader/build_signing.py
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/name_sanity.py
taskcluster/taskgraph/transforms/partner_repack.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/util/signed_artifacts.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-eme-free-repack/kind.yml
@@ -0,0 +1,81 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.release_deps:transforms
+   - taskgraph.transforms.partner_repack:transforms
+   - taskgraph.transforms.release_notifications:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - build-signing
+   - nightly-l10n-signing
+
+job-defaults:
+   name: eme-free-repack
+   description: Release Promotion eme-free repacks
+   run-on-projects: []  # to make sure this never runs as part of CI
+   shipping-product: firefox
+   shipping-phase: promote
+   worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+   worker:
+      docker-image:
+         in-tree: "partner-repack"
+      chain-of-trust: true
+      max-run-time: 7200
+      env:
+         REPACK_MANIFESTS_URL:
+            by-project:
+               mozilla-beta: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
+               mozilla-release: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
+               maple: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
+               default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
+   run:
+      using: mozharness
+      config:
+         - partner_repacks/release_mozilla-release_desktop.py
+      script: mozharness/scripts/desktop_partner_repacks.py
+      job-script: taskcluster/scripts/builder/repackage.sh
+      need-xvfb: false
+      tooltool-downloads: false
+
+jobs:
+   macosx64-nightly:
+      treeherder:
+         symbol: EME
+         platform: macosx64/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: macosx64-nightly
+         build_type: opt
+         artifact_prefix: releng/partner
+         nightly: true
+
+   win32-nightly:
+      treeherder:
+         symbol: EME
+         platform: win32/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: win32-nightly
+         build_type: opt
+         artifact_prefix: releng/partner
+         nightly: true
+
+   win64-nightly:
+      treeherder:
+         symbol: EME
+         platform: win64/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: win64-nightly
+         build_type: opt
+         artifact_prefix: releng/partner
+         nightly: true
--- a/taskcluster/ci/release-partner-repack/kind.yml
+++ b/taskcluster/ci/release-partner-repack/kind.yml
@@ -2,122 +2,79 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.partner_repack:transforms
+   - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.job:transforms
-   - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-   - post-beetmover-dummy
+   - build-signing
+   - nightly-l10n-signing
 
 job-defaults:
+   name: partner-repack
    description: Release Promotion partner repacks
-   worker-type: buildbot-bridge/buildbot-bridge
-   run-on-projects: []
+   run-on-projects: []  # to make sure this never runs as part of CI
    shipping-product: firefox
    shipping-phase: promote
+   worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+   worker:
+      docker-image:
+         in-tree: "partner-repack"
+      chain-of-trust: true
+      max-run-time: 7200
+      env:
+         REPACK_MANIFESTS_URL:
+            by-project:
+               mozilla-beta: "git@github.com:mozilla-partners/repack-manifests.git"
+               mozilla-release: "git@github.com:mozilla-partners/repack-manifests.git"
+               maple: "git@github.com:mozilla-partners/repack-manifests.git"
+               default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
    run:
-      using: buildbot
-      release-promotion: true
-      product: firefox
+      using: mozharness
+      config:
+         - partner_repacks/release_mozilla-release_desktop.py
+      script: mozharness/scripts/desktop_partner_repacks.py
+      job-script: taskcluster/scripts/builder/repackage.sh
+      need-xvfb: false
+      tooltool-downloads: false
 
 jobs:
-   firefox-linux:
-      label: firefox linux partner repacks
-      worker:
-         properties:
-            repack_manifests_url:
-               by-project:
-                  mozilla-beta: "git@github.com:mozilla-partners/repack-manifests.git"
-                  mozilla-release: "git@github.com:mozilla-partners/repack-manifests.git"
-                  default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
-      run:
-         buildername: release-{branch}-firefox-linux_partner_repacks
-
-   firefox-linux64:
-      label: firefox linux64 partner repacks
-      worker:
-         properties:
-            repack_manifests_url:
-               by-project:
-                  mozilla-beta: "git@github.com:mozilla-partners/repack-manifests.git"
-                  mozilla-release: "git@github.com:mozilla-partners/repack-manifests.git"
-                  default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
-      run:
-         buildername: release-{branch}-firefox-linux64_partner_repacks
+   linux-nightly:
+      attributes:
+         build_platform: linux-nightly
+         build_type: opt
+         artifact_prefix: releng/partner
+         nightly: true
 
-   firefox-macosx:
-      label: firefox macosx64 partner repacks
-      worker:
-         properties:
-            repack_manifests_url:
-               by-project:
-                  mozilla-beta: "git@github.com:mozilla-partners/repack-manifests.git"
-                  mozilla-release: "git@github.com:mozilla-partners/repack-manifests.git"
-                  default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
-      run:
-         buildername: release-{branch}-firefox-macosx64_partner_repacks
-
-   firefox-win32:
-      label: firefox win32 partner repacks
-      worker:
-         properties:
-            repack_manifests_url:
-               by-project:
-                  mozilla-beta: "git@github.com:mozilla-partners/repack-manifests.git"
-                  mozilla-release: "git@github.com:mozilla-partners/repack-manifests.git"
-                  default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
-      run:
-         buildername: release-{branch}-firefox-win32_partner_repacks
+   linux64-nightly:
+      attributes:
+         build_platform: linux64-nightly
+         build_type: opt
+         artifact_prefix: releng/partner
+         nightly: true
 
-   firefox-win64:
-      label: firefox win64 partner repacks
-      worker:
-         properties:
-            repack_manifests_url:
-               by-project:
-                  mozilla-beta: "git@github.com:mozilla-partners/repack-manifests.git"
-                  mozilla-release: "git@github.com:mozilla-partners/repack-manifests.git"
-                  default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
-      run:
-         buildername: release-{branch}-firefox-win64_partner_repacks
-
-   firefox-macosx-eme-free:
-      label: firefox macosx64 EME-free repacks
-      worker:
-         properties:
-            repack_manifests_url:
-               by-project:
-                  mozilla-beta: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
-                  mozilla-release: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
-                  default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
-      run:
-         buildername: release-{branch}-firefox-macosx64_partner_repacks
+   macosx64-nightly:
+      attributes:
+         build_platform: macosx64-nightly
+         build_type: opt
+         artifact_prefix: releng/partner
+         nightly: true
 
-   firefox-win32-eme-free:
-      label: firefox win32 EME-free repacks
-      worker:
-         properties:
-            repack_manifests_url:
-               by-project:
-                  mozilla-beta: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
-                  mozilla-release: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
-                  default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
-      run:
-         buildername: release-{branch}-firefox-win32_partner_repacks
+   win32-nightly:
+      attributes:
+         build_platform: win32-nightly
+         build_type: opt
+         artifact_prefix: releng/partner
+         nightly: true
 
-   firefox-win64-eme-free:
-      label: firefox win64 EME-free repacks
-      worker:
-         properties:
-            repack_manifests_url:
-               by-project:
-                  mozilla-beta: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
-                  mozilla-release: "git@github.com:mozilla-partners/mozilla-EME-free-manifest"
-                  default: "git@github.com:mozilla-releng/staging-repack-manifests.git"
-      run:
-         buildername: release-{branch}-firefox-win64_partner_repacks
+   win64-nightly:
+      attributes:
+         build_platform: win64-nightly
+         build_type: opt
+         artifact_prefix: releng/partner
+         nightly: true
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -352,16 +352,20 @@ Generates source for the release
 release-source-signing
 --------------------
 Signs source for the release
 
 release-partner-repack
 ----------------------
 Generates customized versions of releases for partners.
 
+release-eme-free-repack
+----------------------
+Generates customized versions of releases for eme-free repacks.
+
 repackage
 ---------
 Repackage tasks take a signed output and package them up into something suitable
 for shipping to our users. For example, on OSX we return a tarball as the signed output
 and this task would package that up as an Apple Disk Image (.dmg)
 
 repackage-l10n
 --------------
--- a/taskcluster/taskgraph/loader/build_signing.py
+++ b/taskcluster/taskgraph/loader/build_signing.py
@@ -3,27 +3,29 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.loader.single_dep import loader as base_loader
 
 # XXX: This logic should rely in kind.yml. This hasn't been done in the original
 # patch because it required some heavy changes in single_dep.
-LABELS_WHICH_SHOULD_SIGN_CI_BUILDS = (
+NON_NIGHTLY_LABELS_WHICH_SHOULD_SIGN_BUILDS = (
     'build-win32/debug', 'build-win32/opt', 'build-win32/pgo',
     'build-win64/debug', 'build-win64/opt', 'build-win64/pgo',
     'build-win32-devedition/opt', 'build-win64-devedition/opt',
     'build-win64-ccov/debug',
     'release-source-linux64-source/opt',
     'release-source-linux64-fennec-source/opt',
     'release-source-linux64-devedition-source/opt',
+    'release-eme-free-repack-macosx64-nightly',
+    'release-partner-repack-macosx64-nightly',
 )
 
 
 def loader(kind, path, config, params, loaded_tasks):
     jobs = base_loader(kind, path, config, params, loaded_tasks)
 
     for job in jobs:
         dependent_task = job['dependent-task']
         if dependent_task.attributes.get('nightly') or \
-                dependent_task.label in LABELS_WHICH_SHOULD_SIGN_CI_BUILDS:
+                dependent_task.label in NON_NIGHTLY_LABELS_WHICH_SHOULD_SIGN_BUILDS:
             yield job
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -46,20 +46,20 @@ def add_artifacts(config, job, taskdesc,
 
 def docker_worker_add_artifacts(config, job, taskdesc):
     """ Adds an artifact directory to the task """
     add_artifacts(config, job, taskdesc, path='/builds/worker/artifacts/')
 
 
 def generic_worker_add_artifacts(config, job, taskdesc):
     """ Adds an artifact directory to the task """
-    # This ``public/build`` is the location on disk; it doesn't necessarily
-    # mean the artifacts will be public; that is set via the ``artifact_prefix``
-    # attribute.
-    add_artifacts(config, job, taskdesc, path=r'public/build')
+    # The path is the location on disk; it doesn't necessarily
+    # mean the artifacts will be public or private; that is set via the name
+    # attribute in add_artifacts.
+    add_artifacts(config, job, taskdesc, path=get_artifact_prefix(taskdesc))
 
 
 def docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc):
     """Add the GECKO_BASE_* and GECKO_HEAD_* env vars to the worker."""
     env = taskdesc['worker'].setdefault('env', {})
     env.update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REF': config.params['head_rev'],
--- a/taskcluster/taskgraph/transforms/name_sanity.py
+++ b/taskcluster/taskgraph/transforms/name_sanity.py
@@ -21,16 +21,22 @@ def make_label(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
         attr = dep_job.attributes.get
 
         if attr('locale', job.get('locale')):
             template = "{kind}-{locale}-{build_platform}/{build_type}"
         elif attr('l10n_chunk'):
             template = "{kind}-{build_platform}-{l10n_chunk}/{build_type}"
+        elif config.kind.startswith("release-eme-free") or \
+                config.kind.startswith("release-partner-repack"):
+            repack_id = job.get("extra", {}).get("repack_id", None)
+            template = "{kind}-{build_platform}"
+            if repack_id:
+                template += "-{}".format(repack_id.replace('/', '-'))
         else:
             template = "{kind}-{build_platform}/{build_type}"
         job['label'] = template.format(
             kind=config.kind,
             build_platform=attr('build_platform'),
             build_type=attr('build_type'),
             locale=attr('locale', job.get('locale', '')),  # Locale can be absent
             l10n_chunk=attr('l10n_chunk', '')  # Can be empty
--- a/taskcluster/taskgraph/transforms/partner_repack.py
+++ b/taskcluster/taskgraph/transforms/partner_repack.py
@@ -4,20 +4,60 @@
 """
 Transform the partner repack task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.scriptworker import get_release_config
+from taskgraph.util.partners import check_if_partners_enabled
 
 
 transforms = TransformSequence()
 
+transforms.add(check_if_partners_enabled)
+
 
 @transforms.add
 def resolve_properties(config, tasks):
     for task in tasks:
-        for property in ("repack_manifests_url", ):
-            property = "worker.properties.{}".format(property)
+        for property in ("REPACK_MANIFESTS_URL", ):
+            property = "worker.env.{}".format(property)
             resolve_keyed_by(task, property, property, **config.params)
-            yield task
+
+        if task['worker']['env']['REPACK_MANIFESTS_URL'].startswith('git@'):
+            task.setdefault('scopes', []).append(
+                'secrets:get:project/releng/gecko/build/level-{level}/partner-github-ssh'.format(
+                    **config.params
+                )
+            )
+
+        yield task
+
+
+@transforms.add
+def make_label(config, tasks):
+    for task in tasks:
+        task['label'] = "{}-{}".format(config.kind, task['name'])
+        yield task
+
+
+@transforms.add
+def add_command_arguments(config, tasks):
+    release_config = get_release_config(config)
+    for task in tasks:
+        # add the MOZHARNESS_OPTIONS, eg version=61.0, build-number=1, platform=win64
+        task['run']['options'] = [
+            'version={}'.format(release_config['version']),
+            'build-number={}'.format(release_config['build_number']),
+            'platform={}'.format(task['attributes']['build_platform'].split('-')[0]),
+        ]
+
+        # The upstream taskIds are stored a special environment variable, because we want to use
+        # task-reference's to resolve dependencies, but the string handling of MOZHARNESS_OPTIONS
+        # blocks that. It's space-separated string of ids in the end.
+        task['worker']['env']['UPSTREAM_TASKIDS'] = {
+            'task-reference': ' '.join(['<{}>'.format(dep) for dep in task['dependencies']])
+        }
+
+        yield task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -373,16 +373,17 @@ task_description_schema = Schema({
         # the maximum time to run, in seconds
         Required('max-run-time'): int,
 
         # os user groups for test task workers
         Optional('os-groups'): [basestring],
 
         # optional features
         Required('chain-of-trust'): bool,
+        Optional('taskcluster-proxy'): bool,
     }, {
         Required('implementation'): 'buildbot-bridge',
 
         # see
         # https://github.com/mozilla/buildbot-bridge/blob/master/bbb/schemas/payload.yml
         Required('buildername'): basestring,
         Required('sourcestamp'): {
             'branch': basestring,
@@ -458,16 +459,18 @@ task_description_schema = Schema({
         Required('implementation'): 'beetmover',
 
         # the maximum time to run, in seconds
         Required('max-run-time', default=600): int,
 
         # locale key, if this is a locale beetmover job
         Optional('locale'): basestring,
 
+        Optional('partner-public'): bool,
+
         Required('release-properties'): {
             'app-name': basestring,
             'app-version': basestring,
             'branch': basestring,
             'build-id': basestring,
             'hash-type': basestring,
             'platform': basestring,
         },
@@ -962,16 +965,19 @@ def build_generic_worker_payload(config,
         worker['env']['SCCACHE_DISABLE'] = '1'
 
     # currently only support one feature (chain of trust) but this will likely grow
     features = {}
 
     if worker.get('chain-of-trust'):
         features['chainOfTrust'] = True
 
+    if worker.get('taskcluster-proxy'):
+        features['taskclusterProxy'] = True
+
     if features:
         task_def['payload']['features'] = features
 
     # coalesce / superseding
     if 'coalesce' in task:
         task_def['payload']['supersederUrl'] = superseder_url(config, task)
 
 
@@ -1022,16 +1028,18 @@ def build_beetmover_payload(config, task
             'hashType': release_properties['hash-type'],
             'platform': release_properties['platform'],
         },
         'upload_date': config.params['build_date'],
         'upstreamArtifacts':  worker['upstream-artifacts'],
     }
     if worker.get('locale'):
         task_def['payload']['locale'] = worker['locale']
+    if worker.get('partner-public'):
+        task_def['payload']['is_partner_repack_public'] = worker['partner-public']
     if release_config:
         task_def['payload'].update(release_config)
 
 
 @payload_builder('beetmover-cdns')
 def build_beetmover_cdns_payload(config, task, task_def):
     worker = task['worker']
     release_config = get_release_config(config)
--- a/taskcluster/taskgraph/util/signed_artifacts.py
+++ b/taskcluster/taskgraph/util/signed_artifacts.py
@@ -4,16 +4,21 @@
 """
 Defines artifacts to sign before repackage.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 from taskgraph.util.taskcluster import get_artifact_path
 
 
+def is_partner_kind(kind):
+    if kind and kind.startswith(('release-partner', 'release-eme-free')):
+        return True
+
+
 def generate_specifications_of_artifacts_to_sign(
     task, keep_locale_template=True, kind=None
 ):
     build_platform = task.attributes.get('build_platform')
     is_nightly = task.attributes.get('nightly')
     if kind == 'release-source-signing':
         artifacts_specifications = [{
             'artifacts': [
@@ -26,18 +31,22 @@ def generate_specifications_of_artifacts
             'artifacts': [
                 get_artifact_path(task, '{locale}/target.apk'),
             ],
             'formats': ['jar'],
         }]
     # XXX: Mars aren't signed here (on any platform) because internals will be
     # signed at after this stage of the release
     elif 'macosx' in build_platform:
+        if is_partner_kind(kind):
+            extension = 'tar.gz'
+        else:
+            extension = 'dmg'
         artifacts_specifications = [{
-            'artifacts': [get_artifact_path(task, '{locale}/target.dmg')],
+            'artifacts': [get_artifact_path(task, '{{locale}}/target.{}'.format(extension))],
             'formats': ['macapp', 'widevine'],
         }]
     elif 'win' in build_platform:
         artifacts_specifications = [{
             'artifacts': [
                 get_artifact_path(task, '{locale}/setup.exe'),
             ],
             'formats': ['sha2signcode'],
@@ -57,19 +66,33 @@ def generate_specifications_of_artifacts
             'formats': ['gpg', 'widevine'],
         }]
     else:
         raise Exception("Platform not implemented for signing")
 
     if not keep_locale_template:
         artifacts_specifications = _strip_locale_template(artifacts_specifications)
 
+    if is_partner_kind(kind):
+        artifacts_specifications = _strip_widevine_for_partners(artifacts_specifications)
+
     return artifacts_specifications
 
 
 def _strip_locale_template(artifacts_without_locales):
     for spec in artifacts_without_locales:
         for index, artifact in enumerate(spec['artifacts']):
             stripped_artifact = artifact.format(locale='')
             stripped_artifact = stripped_artifact.replace('//', '/')
             spec['artifacts'][index] = stripped_artifact
 
     return artifacts_without_locales
+
+
+def _strip_widevine_for_partners(artifacts_specifications):
+    """ Partner repacks should not resign that's previously signed for fear of breaking partial
+    updates
+    """
+    for spec in artifacts_specifications:
+        if 'widevine' in spec['formats']:
+            spec['formats'].remove('widevine')
+
+    return artifacts_specifications