Bug 1424482 - beetmover: Get rid of balrog_props in favor of task payload r=mtabara draft
authorJohan Lorenzo <jlorenzo@mozilla.com>
Thu, 15 Feb 2018 14:46:19 +0100
changeset 756963 f13fa1b04bbea60e64a680df64bd544144d5f058
parent 756941 d0d3693d9beff5477175a441fdb06e281e8b7f17
push id99603
push userbmo:jlorenzo@mozilla.com
push dateMon, 19 Feb 2018 13:46:21 +0000
reviewersmtabara
bugs1424482
milestone60.0a1
Bug 1424482 - beetmover: Get rid of balrog_props in favor of task payload r=mtabara MozReview-Commit-ID: 5eRwwQ10BQD
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_checksums.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/transforms/beetmover_source.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -17,17 +17,16 @@ from taskgraph.transforms.task import ta
 from voluptuous import Any, Required, Optional
 
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
-    "balrog_props.json",
     "target.common.tests.zip",
     "target.cppunittest.tests.zip",
     "target.crashreporter-symbols.zip",
     "target.json",
     "target.mochitest.tests.zip",
     "target.mozinfo.json",
     "target.reftest.tests.zip",
     "target.talos.tests.zip",
@@ -50,17 +49,16 @@ from voluptuous import Any, Required, Op
     "update/target.complete.mar",
 ]
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
     "target.langpack.xpi",
-    "balrog_props.json",
 ]
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N = [
     "target.complete.mar",
 ]
@@ -87,17 +85,16 @@ from voluptuous import Any, Required, Op
     "en-US/robocop.apk",
     "en-US/target.jsshell.zip",
 ]
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI = [
-    "balrog_props.json",
     "target.common.tests.zip",
     "target.cppunittest.tests.zip",
     "target.json",
     "target.mochitest.tests.zip",
     "target.mozinfo.json",
     "target.reftest.tests.zip",
     "target.talos.tests.zip",
     "target.awsy.tests.zip",
@@ -184,17 +181,17 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
     ],
     'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'linux64-devedition-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'linux-devedition-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'android-x86-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
     'android-x86-old-id-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
     'android-aarch64-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
-    'android-api-16-nightly-l10n': ["balrog_props.json"],
+    'android-api-16-nightly-l10n': [],
     'android-api-16-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
     'android-api-16-old-id-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
     'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'macosx64-devedition-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'win32-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'win32-devedition-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'win64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'win64-devedition-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
@@ -442,16 +439,39 @@ def generate_upstream_artifacts(signing_
             "paths": ["{}/{}".format(artifact_prefix, p)
                       for p in signing_mapping[multi_platform]],
             "locale": "multi",
         }])
 
     return upstream_artifacts
 
 
+def craft_release_properties(config, job):
+    params = config.params
+    build_platform = job['attributes']['build_platform']
+    build_platform = build_platform.replace('-nightly', '')
+    if 'fennec-source' in build_platform:
+        # XXX This case is hardcoded to match the current implementation in beetmover
+        build_platform = 'android-api-16'
+    else:
+        build_platform = build_platform.replace('-source', '')
+
+    app_name = 'Fennec' if 'android' in job['label'] or 'fennec' in job['label'] else 'Firefox'
+
+    return {
+        # XXX Even DevEdition is called Firefox
+        'app-name': app_name,
+        'app-version': str(params['app_version']),
+        'branch': params['project'],
+        'build-id': str(params['moz_build_date']),
+        'hash-type': 'sha512',
+        'platform': build_platform,
+    }
+
+
 @transforms.add
 def make_task_worker(config, jobs):
     for job in jobs:
         valid_beetmover_job = (len(job["dependencies"]) == 2 and
                                any(['signing' in j for j in job['dependencies']]))
         if not valid_beetmover_job:
             raise NotImplementedError("Beetmover must have two dependencies.")
 
@@ -462,19 +482,22 @@ def make_task_worker(config, jobs):
         for dependency in job["dependencies"].keys():
             if 'signing' in dependency:
                 signing_task = dependency
             else:
                 build_task = dependency
 
         signing_task_ref = "<" + str(signing_task) + ">"
         build_task_ref = "<" + str(build_task) + ">"
-        upstream_artifacts = generate_upstream_artifacts(
-            signing_task_ref, build_task_ref, platform, locale
-        )
 
-        worker = {'implementation': 'beetmover',
-                  'upstream-artifacts': upstream_artifacts}
+        worker = {
+            'implementation': 'beetmover',
+            'release-properties': craft_release_properties(config, job),
+            'upstream-artifacts': generate_upstream_artifacts(
+                signing_task_ref, build_task_ref, platform, locale
+            )
+        }
+
         if locale:
             worker["locale"] = locale
         job["worker"] = worker
 
         yield job
--- a/taskcluster/taskgraph/transforms/beetmover_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_checksums.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Transform the checksums signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import craft_release_properties
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
                                          get_beetmover_action_scope)
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
@@ -127,21 +128,16 @@ def generate_upstream_artifacts(refs, pl
         "public/target.checksums.asc",
     ]
 
     upstream_artifacts = [{
         "taskId": {"task-reference": refs["signing"]},
         "taskType": "signing",
         "paths": common_paths,
         "locale": locale or "en-US",
-    }, {
-        "taskId": {"task-reference": refs["beetmover"]},
-        "taskType": "beetmover",
-        "paths": ["public/balrog_props.json"],
-        "locale": locale or "en-US",
     }]
 
     return upstream_artifacts
 
 
 @transforms.add
 def make_beetmover_checksums_worker(config, jobs):
     for job in jobs:
@@ -160,18 +156,21 @@ def make_beetmover_checksums_worker(conf
             if dependency.startswith("beetmover"):
                 refs['beetmover'] = "<{}>".format(dependency)
             else:
                 refs['signing'] = "<{}>".format(dependency)
         if None in refs.values():
             raise NotImplementedError(
                 "Beetmover checksums must have a beetmover and signing dependency!")
 
-        upstream_artifacts = generate_upstream_artifacts(refs,
-                                                         platform, locale)
+        worker = {
+            'implementation': 'beetmover',
+            'release-properties': craft_release_properties(config, job),
+            'upstream-artifacts': generate_upstream_artifacts(
+                refs, platform, locale
+            ),
+        }
 
-        worker = {'implementation': 'beetmover',
-                  'upstream-artifacts': upstream_artifacts}
         if locale:
             worker["locale"] = locale
         job["worker"] = worker
 
         yield job
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Transform the beetmover task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import craft_release_properties
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.partials import (get_balrog_platform_name,
                                      get_partials_artifacts,
                                      get_partials_artifact_map)
 from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
                                          get_beetmover_action_scope,
                                          get_phase)
@@ -32,17 +33,16 @@ logger = logging.getLogger(__name__)
     'win32-devedition-nightly',
 ]
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
-    "balrog_props.json",
     "target.common.tests.zip",
     "target.cppunittest.tests.zip",
     "target.crashreporter-symbols.zip",
     "target.json",
     "target.mochitest.tests.zip",
     "target.mozinfo.json",
     "target.reftest.tests.zip",
     "target.talos.tests.zip",
@@ -58,17 +58,16 @@ logger = logging.getLogger(__name__)
 ]
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
     "target.langpack.xpi",
-    "balrog_props.json",
 ]
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
     r'^(linux(|64)|macosx64)(|-devedition)-nightly$':
@@ -366,23 +365,25 @@ def make_task_worker(config, jobs):
                 build_signing_task = dependency
             else:
                 build_task = "build"
 
         build_task_ref = "<" + str(build_task) + ">"
         build_signing_task_ref = "<" + str(build_signing_task) + ">"
         repackage_task_ref = "<" + str(repackage_task) + ">"
         repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">"
-        upstream_artifacts = generate_upstream_artifacts(
-            build_task_ref, build_signing_task_ref, repackage_task_ref,
-            repackage_signing_task_ref, platform, locale
-        )
 
-        worker = {'implementation': 'beetmover',
-                  'upstream-artifacts': upstream_artifacts}
+        worker = {
+            'implementation': 'beetmover',
+            'release-properties': craft_release_properties(config, job),
+            'upstream-artifacts': generate_upstream_artifacts(
+                build_task_ref, build_signing_task_ref, repackage_task_ref,
+                repackage_signing_task_ref, platform, locale
+            ),
+        }
         if locale:
             worker["locale"] = locale
         job["worker"] = worker
 
         yield job
 
 
 @transforms.add
--- a/taskcluster/taskgraph/transforms/beetmover_source.py
+++ b/taskcluster/taskgraph/transforms/beetmover_source.py
@@ -7,42 +7,24 @@ Transform the beetmover-source task to a
 from __future__ import absolute_import
 
 from taskgraph.transforms.base import TransformSequence
 
 transforms = TransformSequence()
 
 
 @transforms.add
-def tweak_beetmover_source_dependencies_and_upstream_artifacts(config, jobs):
+def remove_build_dependency_in_beetmover_source(config, jobs):
     for job in jobs:
-        # HACK1: instead of grabbing SOURCE file from `release-source` task, we
-        # instead take it along with SOURCE.asc directly from the
-        # `release-source-signing`.
-        #
-        # HACK2: This way, we can just overwrite the `build`
-        # dependency, which at this point still is `release-source` task, with
-        # the actual Nightly en-US linux64 build which contains the
-        # `balrog_props` file we're interested in.
-        #
-        # XXX: this hack should go away by either:
-        # * rewriting beetmover transforms to allow more flexibility in deps
-        # * ditch balrog_props in beetmover and rely on in-tree task payload
+        # XXX: We delete the build dependency because, unlike the other beetmover
+        # tasks, source doesn't depend on any build task at all. This hack should
+        # go away when we rewrite beetmover transforms to allow more flexibility in deps
+        del job['dependencies']['build']
 
-        if job['attributes']['shipping_product'] == 'firefox':
-            job['dependencies']['build'] = u'build-linux64-nightly/opt'
-        elif job['attributes']['shipping_product'] == 'fennec':
-            job['dependencies']['build'] = u'build-android-api-16-nightly/opt'
-        elif job['attributes']['shipping_product'] == 'devedition':
-            job['dependencies']['build'] = u'build-linux64-devedition-nightly/opt'
-        else:
-            raise NotImplemented(
-                "Unknown shipping_product {} for beetmover_source!".format(
-                    job['attributes']['shipping_product']
-                )
-            )
-        upstream_artifacts = job['worker']['upstream-artifacts']
-        for artifact in upstream_artifacts:
-            if artifact['taskType'] == 'build':
-                artifact['paths'].append(u'public/build/balrog_props.json')
-                break
+        all_upstream_artifacts = job['worker']['upstream-artifacts']
+        upstream_artifacts_without_build = [
+            upstream_artifact
+            for upstream_artifact in all_upstream_artifacts
+            if upstream_artifact['taskId']['task-reference'] != '<build>'
+        ]
+        job['worker']['upstream-artifacts'] = upstream_artifacts_without_build
 
         yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -508,16 +508,25 @@ task_description_schema = Schema({
         Required('implementation'): 'beetmover',
 
         # the maximum time to run, in seconds
         Required('max-run-time'): int,
 
         # locale key, if this is a locale beetmover job
         Optional('locale'): basestring,
 
+        Required('release-properties'): {
+            'app-name': basestring,
+            'app-version': basestring,
+            'branch': basestring,
+            'build-id': basestring,
+            'hash-type': basestring,
+            'platform': basestring,
+        },
+
         # list of artifact URLs for the artifacts that should be beetmoved
         Required('upstream-artifacts'): [{
             # taskId of the task with the artifact
             Required('taskId'): taskref_or_string,
 
             # type of signing task (for CoT)
             Required('taskType'): basestring,
 
@@ -1008,21 +1017,30 @@ def build_binary_transparency_payload(co
         ),
     }
 
 
 @payload_builder('beetmover')
 def build_beetmover_payload(config, task, task_def):
     worker = task['worker']
     release_config = get_release_config(config)
+    release_properties = worker['release-properties']
 
     task_def['payload'] = {
         'maxRunTime': worker['max-run-time'],
+        'releaseProperties': {
+            'appName': release_properties['app-name'],
+            'appVersion': release_properties['app-version'],
+            'branch': release_properties['branch'],
+            'buildid': release_properties['build-id'],
+            'hashType': release_properties['hash-type'],
+            'platform': release_properties['platform'],
+        },
         'upload_date': config.params['build_date'],
-        'upstreamArtifacts':  worker['upstream-artifacts']
+        'upstreamArtifacts':  worker['upstream-artifacts'],
     }
     if worker.get('locale'):
         task_def['payload']['locale'] = worker['locale']
     if release_config:
         task_def['payload'].update(release_config)
 
 
 @payload_builder('beetmover-cdns')