Bug 1481121: [release] Use existing release-artifact metadata in beetmover_repackage; r=Callek
authorTom Prince <mozilla@hocat.ca>
Mon, 20 Aug 2018 12:40:21 -0600
changeset 491718 8b061c70417b490d816e997741900dfdea137625
parent 491717 6c58897fc9956b2c47d8a79d073bcca5ff46acf3
child 491719 c284bcf486e6a7b5b450ea07c521073321fe4c91
push id1815
push userffxbld-merge
push dateMon, 15 Oct 2018 10:40:45 +0000
treeherdermozilla-release@18d4c09e9378 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek
bugs1481121
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1481121: [release] Use existing release-artifact metadata in beetmover_repackage; r=Callek Differential Revision: https://phabricator.services.mozilla.com/D3817
taskcluster/ci/beetmover-repackage/kind.yml
taskcluster/taskgraph/loader/single_dep.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py
--- a/taskcluster/ci/beetmover-repackage/kind.yml
+++ b/taskcluster/ci/beetmover-repackage/kind.yml
@@ -10,16 +10,18 @@ transforms:
    - taskgraph.transforms.beetmover_repackage:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - repackage-signing
    - repackage-signing-l10n
    - partials-signing
 
+fake-multi-dep: true
+
 only-for-build-platforms:
    - linux-nightly/opt
    - linux64-nightly/opt
    - macosx64-nightly/opt
    - win32-nightly/opt
    - win64-nightly/opt
    - linux-devedition-nightly/opt
    - linux64-devedition-nightly/opt
--- a/taskcluster/taskgraph/loader/single_dep.py
+++ b/taskcluster/taskgraph/loader/single_dep.py
@@ -24,16 +24,18 @@ def loader(kind, path, config, params, l
     Optional `job-template` kind configuration value, if specified, will be used to
     pass configuration down to the specified transforms used.
     """
     only_platforms = config.get('only-for-build-platforms')
     not_platforms = config.get('not-for-build-platforms')
     only_attributes = config.get('only-for-attributes')
     job_template = config.get('job-template')
 
+    include_parents = config.get('fake-multi-dep')
+
     for task in loaded_tasks:
         if task.kind not in config.get('kind-dependencies', []):
             continue
 
         if only_platforms or not_platforms:
             build_platform = task.attributes.get('build_platform')
             build_type = task.attributes.get('build_type')
             if not build_platform or not build_type:
@@ -46,19 +48,34 @@ def loader(kind, path, config, params, l
 
         if only_attributes:
             config_attrs = set(only_attributes)
             if config_attrs - set(task.attributes):
                 # make sure all attributes exist
                 continue
 
         job = {'dependent-task': task}
+        if include_parents:
+            job['grandparent-tasks'] = _get_grandparent_tasks(task.dependencies, loaded_tasks)
+
         if job_template:
             job.update(copy.deepcopy(job_template))
 
         # copy shipping_product from upstream
         product = task.attributes.get(
             'shipping_product', task.task.get('shipping-product')
         )
         if product:
             job.setdefault('shipping-product', product)
 
         yield job
+
+
+def _get_grandparent_tasks(dependencies, loaded_tasks):
+    parent_tasks = {}
+    for task in loaded_tasks:
+        if task.label in dependencies.values():
+            for name, label in dependencies.items():
+                if label == task.label:
+                    parent_tasks[name] = task
+    if set(parent_tasks.keys()) != set(dependencies.keys()):
+        raise Exception("Missing parent tasks.")
+    return parent_tasks
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -113,39 +113,33 @@ UPSTREAM_ARTIFACT_SIGNED_PATHS = {
         ['target.tar.bz2', 'target.tar.bz2.asc'],
     r'^win(32|64)(|-devedition|-asan-reporter)-nightly(|-l10n)$': ['target.zip'],
 }
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
-UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
-    r'^macosx64(|-devedition)-nightly(|-l10n)$': ['target.dmg'],
-}
+UPSTREAM_ARTIFACT_REPACKAGE_PATHS = [
+    'target.dmg',
+]
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
-UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS = {
-    r'^(linux(|64)|macosx64)(|-devedition|-asan-reporter)-nightly(|-l10n)$':
-        ['target.complete.mar'],
-    r'^win64(|-devedition|-asan-reporter)-nightly(|-l10n)$':
-        ['target.complete.mar', 'target.installer.exe'],
-    r'^win32(|-devedition)-nightly(|-l10n)$': [
-        'target.complete.mar',
-        'target.installer.exe',
-        'target.stub-installer.exe'
-        ],
-}
+UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS = [
+    'target.complete.mar',
+    'target.bz2.complete.mar',
+    'target.installer.exe',
+    'target.stub-installer.exe',
+]
 
 # Compile every regex once at import time
 for dict_ in (
     UPSTREAM_ARTIFACT_UNSIGNED_PATHS, UPSTREAM_ARTIFACT_SIGNED_PATHS,
-    UPSTREAM_ARTIFACT_REPACKAGE_PATHS, UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS,
 ):
     for uncompiled_regex, value in dict_.iteritems():
         compiled_regex = re.compile(uncompiled_regex)
         del dict_[uncompiled_regex]
         dict_[compiled_regex] = value
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
@@ -157,16 +151,18 @@ transforms = TransformSequence()
 taskref_or_string = Any(
     basestring,
     {Required('task-reference'): basestring})
 
 beetmover_description_schema = Schema({
     # the dependent task (object) for this beetmover job, used to inform beetmover.
     Required('dependent-task'): object,
 
+    Required('grandparent-tasks'): object,
+
     # depname is used in taskref's to identify the taskID of the unsigned things
     Required('depname', default='build'): basestring,
 
     # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
     Optional('label'): basestring,
 
     # treeherder is allowed here to override any defaults we use for beetmover.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
@@ -212,38 +208,37 @@ def make_task_description(config, jobs):
                 build_platform=attributes.get('build_platform'),
                 build_type=attributes.get('build_type')
             )
         )
 
         dependent_kind = dep_job.kind
         if dependent_kind == 'repackage-signing-l10n':
             dependent_kind = "repackage-signing"
-        dependencies = {dependent_kind: dep_job.label}
+        dependencies = {dependent_kind: dep_job}
 
         signing_name = "build-signing"
         if job.get('locale'):
             signing_name = "nightly-l10n-signing"
-        dependencies['signing'] = dep_job.dependencies[signing_name]
+        dependencies['signing'] = job['grandparent-tasks'][signing_name]
 
         build_name = "build"
         if job.get('locale'):
             build_name = "unsigned-repack"
-        dependencies["build"] = dep_job.dependencies[build_name]
+        dependencies["build"] = job['grandparent-tasks'][build_name]
 
         # repackage-l10n actually uses the repackage depname here
-        dependencies["repackage"] = dep_job.dependencies["repackage"]
+        dependencies["repackage"] = job['grandparent-tasks']["repackage"]
 
         # If this isn't a direct dependency, it won't be in there.
-        if 'repackage-signing' not in dependencies and \
-                'repackage-signing-l10n' not in dependencies:
+        if 'repackage-signing' not in dependencies:
             repackage_signing_name = "repackage-signing"
             if job.get('locale'):
                 repackage_signing_name = "repackage-signing-l10n"
-            dependencies["repackage-signing"] = dep_job.dependencies[repackage_signing_name]
+            dependencies["repackage-signing"] = job['grandparent-tasks'][repackage_signing_name]
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         if job.get('locale'):
             attributes['locale'] = job['locale']
 
         bucket_scope = get_beetmover_bucket_scope(config)
         action_scope = get_beetmover_action_scope(config)
         phase = get_phase(config)
@@ -259,66 +254,80 @@ def make_task_description(config, jobs):
             'treeherder': treeherder,
             'shipping-phase': job.get('shipping-phase', phase),
             'shipping-product': job.get('shipping-product'),
         }
 
         yield task
 
 
-def generate_upstream_artifacts(job, platform, locale=None, project=None):
+def generate_upstream_artifacts(job, dependencies, platform, locale=None, project=None):
 
     build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
     build_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
     repackage_mapping = UPSTREAM_ARTIFACT_REPACKAGE_PATHS
     repackage_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS
 
     artifact_prefix = get_artifact_prefix(job)
     if locale:
         artifact_prefix = '{}/{}'.format(artifact_prefix, locale)
         platform = "{}-l10n".format(platform)
 
     upstream_artifacts = []
 
-    for task_type, cot_type, mapping in [
-        ("build", "build", build_mapping),
-        ("signing", "signing", build_signing_mapping),
-        ('repackage', 'repackage', repackage_mapping),
-        ('repackage-signing', 'repackage', repackage_signing_mapping),
+    for task_type, mapping in [
+        ("build", build_mapping),
+        ("signing", build_signing_mapping),
     ]:
         platform_was_previously_matched_by_regex = None
         for platform_regex, paths in mapping.iteritems():
             if platform_regex.match(platform) is not None:
                 _check_platform_matched_only_one_regex(
                     task_type, platform, platform_was_previously_matched_by_regex, platform_regex
                 )
                 platform_was_previously_matched_by_regex = platform_regex
                 if paths:
                     usable_paths = paths[:]
 
-                    use_stub = job["attributes"].get('stub-installer')
-                    if not use_stub:
-                        if 'target.stub-installer.exe' in usable_paths:
-                            usable_paths.remove('target.stub-installer.exe')
                     if 'target.langpack.xpi' in usable_paths and \
                             not project == "mozilla-central":
                         # XXX This is only beetmoved for m-c nightlies.
                         # we should determine this better
                         usable_paths.remove('target.langpack.xpi')
+
                         if not len(usable_paths):
                             # We may have removed our only path.
                             continue
+
                     upstream_artifacts.append({
                         "taskId": {"task-reference": "<{}>".format(task_type)},
-                        "taskType": cot_type,
-                        "paths": ["{}/{}".format(artifact_prefix, path)
-                                  for path in usable_paths],
+                        "taskType": task_type,
+                        "paths": ["{}/{}".format(artifact_prefix, path) for path in usable_paths],
                         "locale": locale or "en-US",
                     })
 
+    for task_type, cot_type, paths in [
+        ('repackage', 'repackage', repackage_mapping),
+        ('repackage-signing', 'repackage', repackage_signing_mapping),
+    ]:
+        paths = ["{}/{}".format(artifact_prefix, path) for path in paths]
+        paths = [
+            path for path in paths
+            if path in dependencies[task_type].release_artifacts]
+
+        if not paths:
+            continue
+
+        upstream_artifacts.append({
+            "taskId": {"task-reference": "<{}>".format(task_type)},
+            "taskType": cot_type,
+            "paths": paths,
+            "locale": locale or "en-US",
+        })
+
     return upstream_artifacts
 
 
 def generate_partials_upstream_artifacts(job, artifacts, platform, locale=None):
     artifact_prefix = get_artifact_prefix(job)
     if locale and locale != 'en-US':
         artifact_prefix = '{}/{}'.format(artifact_prefix, locale)
 
@@ -366,17 +375,17 @@ def make_task_worker(config, jobs):
             raise NotImplementedError(
                 "{}: Beetmover_repackage must have five dependencies.".format(job['label'])
             )
 
         locale = job["attributes"].get("locale")
         platform = job["attributes"]["build_platform"]
 
         upstream_artifacts = generate_upstream_artifacts(
-            job, platform, locale,
+            job, job['dependencies'], platform, locale,
             project=config.params['project']
         )
 
         worker = {
             'implementation': 'beetmover',
             'release-properties': craft_release_properties(config, job),
             'upstream-artifacts': upstream_artifacts,
         }
@@ -438,8 +447,18 @@ def make_partials_artifacts(config, jobs
                 if artifact_map[artifact].get(rel_attr):
                     artifact_extra[rel_attr] = artifact_map[artifact][rel_attr]
             extra.append(artifact_extra)
 
         job.setdefault('extra', {})
         job['extra']['partials'] = extra
 
         yield job
+
+
+@transforms.add
+def convert_deps(config, jobs):
+    for job in jobs:
+        job['dependencies'] = {
+            name: dep_job.label
+            for name, dep_job in job['dependencies'].items()
+        }
+        yield job
--- a/taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py
@@ -30,14 +30,15 @@ def make_beetmover_description(config, j
 
         treeherder = {
             'symbol': join_symbol(group, symbol),
         }
 
         beet_description = {
             'label': job['label'],
             'dependent-task': dep_job,
+            'grandparent-tasks': job['grandparent-tasks'],
             'treeherder': treeherder,
             'locale': locale,
             'shipping-phase': job.get('shipping-phase'),
             'shipping-product': job.get('shipping-product'),
         }
         yield beet_description