Bug 1357867 - Add balrog and beetmover support to cross compiled OSX, allow dependencies on repackage jobs. r=dustin
authorJustin Wood <Callek@gmail.com>
Wed, 19 Apr 2017 15:49:55 -0400
changeset 566280 feb371151541d2a14a138926f2f277f80bff781f
parent 566279 598dd73ca7f3c58fc957607c10089709af3bbdf4
child 566281 c631c917016bc2f283305f8df08336fc9bee1445
push id55180
push userjjong@mozilla.com
push dateFri, 21 Apr 2017 09:36:13 +0000
reviewersdustin
bugs1357867
milestone55.0a1
Bug 1357867 - Add balrog and beetmover support to cross compiled OSX, allow dependencies on repackage jobs. r=dustin MozReview-Commit-ID: Frc0sMaC1o0
taskcluster/ci/balrog/kind.yml
taskcluster/ci/beetmover-repackage/kind.yml
taskcluster/ci/beetmover/kind.yml
taskcluster/ci/checksums-signing/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/loader/single_dep.py
taskcluster/taskgraph/transforms/balrog.py
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/util/scriptworker.py
--- a/taskcluster/ci/balrog/kind.yml
+++ b/taskcluster/ci/balrog/kind.yml
@@ -6,11 +6,12 @@ loader: taskgraph.loader.single_dep:load
 
 transforms:
    - taskgraph.transforms.balrog:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - beetmover
   - beetmover-l10n
+  - beetmover-repackage
 
 only-for-attributes:
   - nightly
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover-repackage/kind.yml
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.beetmover_repackage:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - repackage
+
+only-for-build-platforms:
+  - macosx64-nightly/opt
--- a/taskcluster/ci/beetmover/kind.yml
+++ b/taskcluster/ci/beetmover/kind.yml
@@ -8,8 +8,11 @@ transforms:
    - taskgraph.transforms.beetmover:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - build-signing
 
 only-for-attributes:
   - nightly
+
+not-for-build-platforms:
+   - macosx64-nightly/opt
--- a/taskcluster/ci/checksums-signing/kind.yml
+++ b/taskcluster/ci/checksums-signing/kind.yml
@@ -6,11 +6,12 @@ loader: taskgraph.loader.single_dep:load
 
 transforms:
    - taskgraph.transforms.checksums_signing:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - beetmover
   - beetmover-l10n
+  - beetmover-repackage
 
 only-for-attributes:
   - nightly
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -179,16 +179,24 @@ beetmover-l10n
 --------------
 
 Beetmover L10n, takes specific artifacts, "Beets", and pushes them to a location outside
 of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
 process determines the final location and a "pretty" name (versioned product name)
 This separate kind uses logic specific to localized artifacts, such as including
 the language in the final artifact names.
 
+beetmover-repackage
+---------
+
+Beetmover-repackage is beetmover but for tasks that need an intermediate step
+between signing and packaging, such as OSX. For more details see the definitions
+of the Beetmover kind above and the repackage kind below.
+
+
 checksums-signing
 ----------------
 Checksums-signing take as input the checksums file generated by beetmover tasks
 and sign it via the signing scriptworkers. Returns the same file signed and
 additional detached signature.
 
 beetmover-checksums
 -------------------
--- a/taskcluster/taskgraph/loader/single_dep.py
+++ b/taskcluster/taskgraph/loader/single_dep.py
@@ -7,40 +7,46 @@ from __future__ import absolute_import, 
 import copy
 
 
 def loader(kind, path, config, params, loaded_tasks):
     """
     Load tasks based on the jobs dependant kinds.
 
     The `only-for-build-platforms` kind configuration, if specified, will limit
-    the build platforms for which a job will be created.
+    the build platforms for which a job will be created. Alternatively there is
+    'not-for-build-platforms' kind configuration which will be consulted only after
+    'only-for-build-platforms' is checked (if present), and omit any jobs where the
+    build platform matches.
 
     Optional `only-for-attributes` kind configuration, if specified, will limit
     the jobs chosen to ones which have the specified attribute, with the specified
     value.
 
     Optional `job-template` kind configuration value, if specified, will be used to
     pass configuration down to the specified transforms used.
     """
     only_platforms = config.get('only-for-build-platforms')
+    not_platforms = config.get('not-for-build-platforms')
     only_attributes = config.get('only-for-attributes')
     job_template = config.get('job-template')
 
     for task in loaded_tasks:
         if task.kind not in config.get('kind-dependencies', []):
             continue
 
-        if only_platforms:
+        if only_platforms or not_platforms:
             build_platform = task.attributes.get('build_platform')
             build_type = task.attributes.get('build_type')
             if not build_platform or not build_type:
                 continue
             platform = "{}/{}".format(build_platform, build_type)
-            if platform not in only_platforms:
+            if only_platforms and platform not in only_platforms:
+                continue
+            elif not_platforms and platform in not_platforms:
                 continue
 
         if only_attributes:
             config_attrs = set(only_attributes)
             if config_attrs - set(task.attributes):
                 # make sure all attributes exist
                 continue
 
--- a/taskcluster/taskgraph/transforms/balrog.py
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -47,17 +47,18 @@ def validate(config, jobs):
         yield validate_schema(
             balrog_description_schema, job,
             "In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
 
 
 @transforms.add
 def skip_unsigned_beets(config, jobs):
     for job in jobs:
-        if 'signing' not in job['dependent-task'].label:
+        if ('signing' not in job['dependent-task'].label and
+                'beetmover-repackage' not in job['dependent-task'].label):
             # Skip making a balrog task for this
             continue
         yield job
 
 
 @transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -100,50 +100,56 @@ from voluptuous import Any, Required, Op
 ]
 
 
 UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
     'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-api-15-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
-    'macosx64-nightly': [],
+    'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
 
     'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'android-x86-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
     'android-api-15-nightly-l10n': ["balrog_props.json"],
     'android-api-15-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
-    'macosx64-nightly-l10n': [],
+    'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
 }
 UPSTREAM_ARTIFACT_SIGNED_PATHS = {
     'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'android-x86-nightly': ["en-US/target.apk"],
     'android-api-15-nightly': ["en-US/target.apk"],
-    'macosx64-nightly': [],
+    'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.dmg",
+        "target.dmg.asc",
+    ],
 
     'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'android-x86-nightly-multi': ["target.apk"],
     'android-api-15-nightly-l10n': ["target.apk"],
     'android-api-15-nightly-multi': ["target.apk"],
-    'macosx64-nightly-l10n': [],
+    'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
+        "target.dmg",
+        "target.dmg.asc",
+    ],
 }
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
 transforms = TransformSequence()
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -0,0 +1,244 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
+                                         get_beetmover_action_scope)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Any, Required, Optional
+
+import logging
+logger = logging.getLogger(__name__)
+
+# For developers: if you are adding any new artifacts here that need to be
+# transfered to S3, please be aware you also need to follow-up with patch in
+# the actual beetmoverscript logic that lies under
+# https://github.com/mozilla-releng/beetmoverscript/. See example in bug
+# 1348286
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
+    "balrog_props.json",
+    "target.common.tests.zip",
+    "target.cppunittest.tests.zip",
+    "target.crashreporter-symbols.zip",
+    "target.json",
+    "target.mochitest.tests.zip",
+    "target.mozinfo.json",
+    "target.reftest.tests.zip",
+    "target.talos.tests.zip",
+    "target.awsy.tests.zip",
+    "target.test_packages.json",
+    "target.txt",
+    "target.web-platform.tests.zip",
+    "target.xpcshell.tests.zip",
+    "target_info.txt",
+    "target.jsshell.zip",
+    "mozharness.zip",
+    "target.langpack.xpi",
+    "host/bin/mar",
+    "host/bin/mbsdiff",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
+    "update/target.complete.mar",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
+    "target.langpack.xpi",
+    "balrog_props.json",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N = [
+    "target.complete.mar",
+]
+
+UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
+    'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+    'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
+}
+UPSTREAM_ARTIFACT_SIGNED_PATHS = {
+    'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US,
+    'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N,
+}
+UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
+    'macosx64-nightly': ["target.dmg"],
+    'macosx64-nightly-l10n': ["target.dmg"],
+}
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+beetmover_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-task'): object,
+
+    # depname is used in taskref's to identify the taskID of the unsigned things
+    Required('depname', default='build'): basestring,
+
+    # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
+    Optional('label'): basestring,
+
+    # treeherder is allowed here to override any defaults we use for beetmover.  See
+    # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+    # below transforms for defaults of various values.
+    Optional('treeherder'): task_description_schema['treeherder'],
+
+    # locale is passed only for l10n beetmoving
+    Optional('locale'): basestring,
+})
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        yield validate_schema(
+            beetmover_description_schema, job,
+            "In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'tc(BM-R)')
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 1)
+        treeherder.setdefault('kind', 'build')
+        label = job.get('label', "beetmover-{}".format(dep_job.label))
+        dependent_kind = str(dep_job.kind)
+        dependencies = {dependent_kind: dep_job.label}
+
+        # macosx nightly builds depend on repackage which use in tree docker
+        # images and thus have two dependencies
+        # change the signing_dependencies to be use the ones in
+        docker_dependencies = {"docker-image":
+                                dep_job.dependencies["docker-image"]
+                               }
+        dependencies.update(docker_dependencies)
+        signing_dependencies = {"build-signing":
+                                dep_job.dependencies["build-signing"]
+                               }
+        dependencies.update(signing_dependencies)
+        repackage_dependencies = {"repackage":
+                                  dep_job.label
+                                 }
+        dependencies.update(repackage_dependencies)
+        build_dependencies = {"build":
+                              dep_job.dependencies["build-signing"].replace("signing-","build-")
+                             }
+        dependencies.update(build_dependencies)
+
+        attributes = {
+            'nightly': dep_job.attributes.get('nightly', False),
+            'build_platform': dep_job.attributes.get('build_platform'),
+            'build_type': dep_job.attributes.get('build_type'),
+        }
+        if job.get('locale'):
+            attributes['locale'] = job['locale']
+
+        bucket_scope = get_beetmover_bucket_scope(config)
+        action_scope = get_beetmover_action_scope(config)
+
+        task = {
+            'label': label,
+            'description': "{} Beetmover".format(
+                dep_job.task["metadata"]["description"]),
+            'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
+            'scopes': [bucket_scope, action_scope],
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+        }
+
+        yield task
+
+
+def generate_upstream_artifacts(signing_task_ref, build_task_ref,
+                                repackage_task_ref, platform,
+                                locale=None):
+
+    signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
+    build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
+    repackage_mapping = UPSTREAM_ARTIFACT_REPACKAGE_PATHS
+
+    artifact_prefix = 'public/build'
+    if locale:
+        artifact_prefix = 'public/build/{}'.format(locale)
+        platform = "{}-l10n".format(platform)
+
+    upstream_artifacts = [{
+        "taskId": {"task-reference": build_task_ref},
+        "taskType": "build",
+        "paths": ["{}/{}".format(artifact_prefix, p)
+                  for p in build_mapping[platform]],
+        "locale": locale or "en-US",
+        }, {
+        "taskId": {"task-reference": signing_task_ref},
+        "taskType": "signing",
+        "paths": ["{}/{}".format(artifact_prefix, p)
+                  for p in signing_mapping[platform]],
+        "locale": locale or "en-US",
+        }, {
+        "taskId": {"task-reference": repackage_task_ref},
+        "taskType": "repackage",
+        "paths": ["{}/{}".format(artifact_prefix, p)
+                  for p in repackage_mapping[platform]],
+        "locale": locale or "en-US",
+    }]
+
+    return upstream_artifacts
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        valid_beetmover_job = (len(job["dependencies"]) == 4 and
+                               any(['repackage' in j for j in job['dependencies']]))
+        if not valid_beetmover_job:
+            raise NotImplementedError("Beetmover_repackage must have four dependencies.")
+
+        locale = job["attributes"].get("locale")
+        platform = job["attributes"]["build_platform"]
+        build_task = None
+        repackage_task = None
+        signing_task = None
+        for dependency in job["dependencies"].keys():
+            if 'repackage' in dependency:
+                repackage_task = dependency
+            elif 'signing' in dependency:
+                signing_task = dependency
+            else:
+                build_task = "build"
+
+        signing_task_ref = "<" + str(signing_task) + ">"
+        build_task_ref = "<" + str(build_task) + ">"
+        repackage_task_ref = "<" + str(repackage_task) + ">"
+        upstream_artifacts = generate_upstream_artifacts(
+            signing_task_ref, build_task_ref, repackage_task_ref, platform, locale
+        )
+
+        worker = {'implementation': 'beetmover',
+                  'upstream-artifacts': upstream_artifacts}
+        if locale:
+            worker["locale"] = locale
+        job["worker"] = worker
+
+        yield job
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -79,16 +79,17 @@ BEETMOVER_RELEASE_TARGET_TASKS = set([
 """Map beetmover tasks aliases to sets of target task methods.
 
 This is a list of list-pairs, for ordering.
 """
 BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK = [[
     'all-nightly-tasks', set([
         'nightly_fennec',
         'nightly_linux',
+        'nightly_macosx',
         'mozilla_beta_tasks',
         'mozilla_release_tasks',
     ])
 ], [
     'all-release-tasks', BEETMOVER_RELEASE_TARGET_TASKS
 ]]
 
 """Map the beetmover scope aliases to the actual scopes.