Bug 1357867 - Add balrog and beetmover support to cross compiled OSX, allow dependencies on repackage jobs. r=dustin
authorJustin Wood <Callek@gmail.com>
Wed, 19 Apr 2017 15:49:55 -0400
changeset 354175 feb371151541d2a14a138926f2f277f80bff781f
parent 354174 598dd73ca7f3c58fc957607c10089709af3bbdf4
child 354176 c631c917016bc2f283305f8df08336fc9bee1445
push id31687
push usercbook@mozilla.com
push dateFri, 21 Apr 2017 08:45:35 +0000
treeherdermozilla-central@7b43acb94854 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1357867
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1357867 - Add balrog and beetmover support to cross compiled OSX, allow dependencies on repackage jobs. r=dustin MozReview-Commit-ID: Frc0sMaC1o0
taskcluster/ci/balrog/kind.yml
taskcluster/ci/beetmover-repackage/kind.yml
taskcluster/ci/beetmover/kind.yml
taskcluster/ci/checksums-signing/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/loader/single_dep.py
taskcluster/taskgraph/transforms/balrog.py
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/util/scriptworker.py
--- a/taskcluster/ci/balrog/kind.yml
+++ b/taskcluster/ci/balrog/kind.yml
@@ -6,11 +6,12 @@ loader: taskgraph.loader.single_dep:load
 
 transforms:
    - taskgraph.transforms.balrog:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - beetmover
   - beetmover-l10n
+  - beetmover-repackage
 
 only-for-attributes:
   - nightly
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover-repackage/kind.yml
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.beetmover_repackage:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - repackage
+
+only-for-build-platforms:
+  - macosx64-nightly/opt
--- a/taskcluster/ci/beetmover/kind.yml
+++ b/taskcluster/ci/beetmover/kind.yml
@@ -8,8 +8,11 @@ transforms:
    - taskgraph.transforms.beetmover:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - build-signing
 
 only-for-attributes:
   - nightly
+
+not-for-build-platforms:
+   - macosx64-nightly/opt
--- a/taskcluster/ci/checksums-signing/kind.yml
+++ b/taskcluster/ci/checksums-signing/kind.yml
@@ -6,11 +6,12 @@ loader: taskgraph.loader.single_dep:load
 
 transforms:
    - taskgraph.transforms.checksums_signing:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - beetmover
   - beetmover-l10n
+  - beetmover-repackage
 
 only-for-attributes:
   - nightly
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -179,16 +179,24 @@ beetmover-l10n
 --------------
 
 Beetmover L10n, takes specific artifacts, "Beets", and pushes them to a location outside
 of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
 process determines the final location and a "pretty" name (versioned product name)
 This separate kind uses logic specific to localized artifacts, such as including
 the language in the final artifact names.
 
+beetmover-repackage
+---------
+
+Beetmover-repackage is beetmover but for tasks that need an intermediate step
+between signing and packaging, such as OSX. For more details see the definitions
+of the Beetmover kind above and the repackage kind below.
+
+
 checksums-signing
 ----------------
 Checksums-signing take as input the checksums file generated by beetmover tasks
 and sign it via the signing scriptworkers. Returns the same file signed and
 additional detached signature.
 
 beetmover-checksums
 -------------------
--- a/taskcluster/taskgraph/loader/single_dep.py
+++ b/taskcluster/taskgraph/loader/single_dep.py
@@ -7,40 +7,46 @@ from __future__ import absolute_import, 
 import copy
 
 
 def loader(kind, path, config, params, loaded_tasks):
     """
     Load tasks based on the jobs dependant kinds.
 
     The `only-for-build-platforms` kind configuration, if specified, will limit
-    the build platforms for which a job will be created.
+    the build platforms for which a job will be created. Alternatively there is
+    'not-for-build-platforms' kind configuration which will be consulted only after
+    'only-for-build-platforms' is checked (if present), and omit any jobs where the
+    build platform matches.
 
     Optional `only-for-attributes` kind configuration, if specified, will limit
     the jobs chosen to ones which have the specified attribute, with the specified
     value.
 
     Optional `job-template` kind configuration value, if specified, will be used to
     pass configuration down to the specified transforms used.
     """
     only_platforms = config.get('only-for-build-platforms')
+    not_platforms = config.get('not-for-build-platforms')
     only_attributes = config.get('only-for-attributes')
     job_template = config.get('job-template')
 
     for task in loaded_tasks:
         if task.kind not in config.get('kind-dependencies', []):
             continue
 
-        if only_platforms:
+        if only_platforms or not_platforms:
             build_platform = task.attributes.get('build_platform')
             build_type = task.attributes.get('build_type')
             if not build_platform or not build_type:
                 continue
             platform = "{}/{}".format(build_platform, build_type)
-            if platform not in only_platforms:
+            if only_platforms and platform not in only_platforms:
+                continue
+            elif not_platforms and platform in not_platforms:
                 continue
 
         if only_attributes:
             config_attrs = set(only_attributes)
             if config_attrs - set(task.attributes):
                 # make sure all attributes exist
                 continue
 
--- a/taskcluster/taskgraph/transforms/balrog.py
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -47,17 +47,18 @@ def validate(config, jobs):
         yield validate_schema(
             balrog_description_schema, job,
             "In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
 
 
 @transforms.add
 def skip_unsigned_beets(config, jobs):
     for job in jobs:
-        if 'signing' not in job['dependent-task'].label:
+        if ('signing' not in job['dependent-task'].label and
+                'beetmover-repackage' not in job['dependent-task'].label):
             # Skip making a balrog task for this
             continue
         yield job
 
 
 @transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -100,50 +100,56 @@ from voluptuous import Any, Required, Op
 ]
 
 
 UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
     'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-api-15-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
-    'macosx64-nightly': [],
+    'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
 
     'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'android-x86-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
     'android-api-15-nightly-l10n': ["balrog_props.json"],
     'android-api-15-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
-    'macosx64-nightly-l10n': [],
+    'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
 }
 UPSTREAM_ARTIFACT_SIGNED_PATHS = {
     'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'android-x86-nightly': ["en-US/target.apk"],
     'android-api-15-nightly': ["en-US/target.apk"],
-    'macosx64-nightly': [],
+    'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.dmg",
+        "target.dmg.asc",
+    ],
 
     'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'android-x86-nightly-multi': ["target.apk"],
     'android-api-15-nightly-l10n': ["target.apk"],
     'android-api-15-nightly-multi': ["target.apk"],
-    'macosx64-nightly-l10n': [],
+    'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
+        "target.dmg",
+        "target.dmg.asc",
+    ],
 }
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
 transforms = TransformSequence()
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -0,0 +1,244 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
+                                         get_beetmover_action_scope)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Any, Required, Optional
+
+import logging
+logger = logging.getLogger(__name__)
+
+# For developers: if you are adding any new artifacts here that need to be
+# transfered to S3, please be aware you also need to follow-up with patch in
+# the actual beetmoverscript logic that lies under
+# https://github.com/mozilla-releng/beetmoverscript/. See example in bug
+# 1348286
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
+    "balrog_props.json",
+    "target.common.tests.zip",
+    "target.cppunittest.tests.zip",
+    "target.crashreporter-symbols.zip",
+    "target.json",
+    "target.mochitest.tests.zip",
+    "target.mozinfo.json",
+    "target.reftest.tests.zip",
+    "target.talos.tests.zip",
+    "target.awsy.tests.zip",
+    "target.test_packages.json",
+    "target.txt",
+    "target.web-platform.tests.zip",
+    "target.xpcshell.tests.zip",
+    "target_info.txt",
+    "target.jsshell.zip",
+    "mozharness.zip",
+    "target.langpack.xpi",
+    "host/bin/mar",
+    "host/bin/mbsdiff",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
+    "update/target.complete.mar",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
+    "target.langpack.xpi",
+    "balrog_props.json",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N = [
+    "target.complete.mar",
+]
+
+UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
+    'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+    'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
+}
+UPSTREAM_ARTIFACT_SIGNED_PATHS = {
+    'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US,
+    'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N,
+}
+UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
+    'macosx64-nightly': ["target.dmg"],
+    'macosx64-nightly-l10n': ["target.dmg"],
+}
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+beetmover_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-task'): object,
+
+    # depname is used in taskref's to identify the taskID of the unsigned things
+    Required('depname', default='build'): basestring,
+
+    # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
+    Optional('label'): basestring,
+
+    # treeherder is allowed here to override any defaults we use for beetmover.  See
+    # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+    # below transforms for defaults of various values.
+    Optional('treeherder'): task_description_schema['treeherder'],
+
+    # locale is passed only for l10n beetmoving
+    Optional('locale'): basestring,
+})
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        yield validate_schema(
+            beetmover_description_schema, job,
+            "In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'tc(BM-R)')
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 1)
+        treeherder.setdefault('kind', 'build')
+        label = job.get('label', "beetmover-{}".format(dep_job.label))
+        dependent_kind = str(dep_job.kind)
+        dependencies = {dependent_kind: dep_job.label}
+
+        # macosx nightly builds depend on repackage which use in tree docker
+        # images and thus have two dependencies
+        # change the signing_dependencies to be use the ones in
+        docker_dependencies = {"docker-image":
+                                dep_job.dependencies["docker-image"]
+                               }
+        dependencies.update(docker_dependencies)
+        signing_dependencies = {"build-signing":
+                                dep_job.dependencies["build-signing"]
+                               }
+        dependencies.update(signing_dependencies)
+        repackage_dependencies = {"repackage":
+                                  dep_job.label
+                                 }
+        dependencies.update(repackage_dependencies)
+        build_dependencies = {"build":
+                              dep_job.dependencies["build-signing"].replace("signing-","build-")
+                             }
+        dependencies.update(build_dependencies)
+
+        attributes = {
+            'nightly': dep_job.attributes.get('nightly', False),
+            'build_platform': dep_job.attributes.get('build_platform'),
+            'build_type': dep_job.attributes.get('build_type'),
+        }
+        if job.get('locale'):
+            attributes['locale'] = job['locale']
+
+        bucket_scope = get_beetmover_bucket_scope(config)
+        action_scope = get_beetmover_action_scope(config)
+
+        task = {
+            'label': label,
+            'description': "{} Beetmover".format(
+                dep_job.task["metadata"]["description"]),
+            'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
+            'scopes': [bucket_scope, action_scope],
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+        }
+
+        yield task
+
+
+def generate_upstream_artifacts(signing_task_ref, build_task_ref,
+                                repackage_task_ref, platform,
+                                locale=None):
+
+    signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
+    build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
+    repackage_mapping = UPSTREAM_ARTIFACT_REPACKAGE_PATHS
+
+    artifact_prefix = 'public/build'
+    if locale:
+        artifact_prefix = 'public/build/{}'.format(locale)
+        platform = "{}-l10n".format(platform)
+
+    upstream_artifacts = [{
+        "taskId": {"task-reference": build_task_ref},
+        "taskType": "build",
+        "paths": ["{}/{}".format(artifact_prefix, p)
+                  for p in build_mapping[platform]],
+        "locale": locale or "en-US",
+        }, {
+        "taskId": {"task-reference": signing_task_ref},
+        "taskType": "signing",
+        "paths": ["{}/{}".format(artifact_prefix, p)
+                  for p in signing_mapping[platform]],
+        "locale": locale or "en-US",
+        }, {
+        "taskId": {"task-reference": repackage_task_ref},
+        "taskType": "repackage",
+        "paths": ["{}/{}".format(artifact_prefix, p)
+                  for p in repackage_mapping[platform]],
+        "locale": locale or "en-US",
+    }]
+
+    return upstream_artifacts
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        valid_beetmover_job = (len(job["dependencies"]) == 4 and
+                               any(['repackage' in j for j in job['dependencies']]))
+        if not valid_beetmover_job:
+            raise NotImplementedError("Beetmover_repackage must have four dependencies.")
+
+        locale = job["attributes"].get("locale")
+        platform = job["attributes"]["build_platform"]
+        build_task = None
+        repackage_task = None
+        signing_task = None
+        for dependency in job["dependencies"].keys():
+            if 'repackage' in dependency:
+                repackage_task = dependency
+            elif 'signing' in dependency:
+                signing_task = dependency
+            else:
+                build_task = "build"
+
+        signing_task_ref = "<" + str(signing_task) + ">"
+        build_task_ref = "<" + str(build_task) + ">"
+        repackage_task_ref = "<" + str(repackage_task) + ">"
+        upstream_artifacts = generate_upstream_artifacts(
+            signing_task_ref, build_task_ref, repackage_task_ref, platform, locale
+        )
+
+        worker = {'implementation': 'beetmover',
+                  'upstream-artifacts': upstream_artifacts}
+        if locale:
+            worker["locale"] = locale
+        job["worker"] = worker
+
+        yield job
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -79,16 +79,17 @@ BEETMOVER_RELEASE_TARGET_TASKS = set([
 """Map beetmover tasks aliases to sets of target task methods.
 
 This is a list of list-pairs, for ordering.
 """
 BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK = [[
     'all-nightly-tasks', set([
         'nightly_fennec',
         'nightly_linux',
+        'nightly_macosx',
         'mozilla_beta_tasks',
         'mozilla_release_tasks',
     ])
 ], [
     'all-release-tasks', BEETMOVER_RELEASE_TARGET_TASKS
 ]]
 
 """Map the beetmover scope aliases to the actual scopes.