Bug 1499254 - beetmover_repackage{,_l10n} multi-dep. r=tomprince
authorAki Sasaki <asasaki@mozilla.com>
Tue, 23 Oct 2018 21:12:39 +0000
changeset 491031 848914c4c934be47a40a70fc09c5c26cd8df0ec4
parent 491030 2eea4d6c1597ce9a997eca8bc5fdb78119d7aced
child 491032 f99c22e6f6b1e8820eb20936aee362dc86dbd454
push id247
push userfmarier@mozilla.com
push dateSat, 27 Oct 2018 01:06:44 +0000
reviewerstomprince
bugs1499254
milestone65.0a1
Bug 1499254 - beetmover_repackage{,_l10n} multi-dep. r=tomprince Depends on D9198 Differential Revision: https://phabricator.services.mozilla.com/D9200
taskcluster/ci/beetmover-repackage/kind.yml
taskcluster/taskgraph/loader/multi_dep.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py
--- a/taskcluster/ci/beetmover-repackage/kind.yml
+++ b/taskcluster/ci/beetmover-repackage/kind.yml
@@ -1,35 +1,46 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-loader: taskgraph.loader.single_dep:loader
+loader: taskgraph.loader.multi_dep:loader
+
+group-by: single-locale
 
 transforms:
-   - taskgraph.transforms.name_sanity:transforms
-   - taskgraph.transforms.beetmover_repackage_l10n:transforms
-   - taskgraph.transforms.beetmover_repackage:transforms
-   - taskgraph.transforms.task:transforms
+    - taskgraph.transforms.name_sanity:transforms
+    - taskgraph.transforms.beetmover_repackage_l10n:transforms
+    - taskgraph.transforms.beetmover_repackage:transforms
+    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-   - repackage-signing
-   - repackage-signing-l10n
-   - partials-signing
+    - build
+    - build-signing
+    - repackage
+    - repackage-signing
+    - nightly-l10n
+    - nightly-l10n-signing
+    - repackage-l10n
+    - repackage-signing-l10n
+    - partials
+    - partials-signing
 
-fake-multi-dep: true
+primary-dependency:
+    - repackage-signing-l10n
+    - repackage-signing
 
 only-for-build-platforms:
-   - linux-nightly/opt
-   - linux64-nightly/opt
-   - macosx64-nightly/opt
-   - win32-nightly/opt
-   - win64-nightly/opt
-   - linux-devedition-nightly/opt
-   - linux64-devedition-nightly/opt
-   - macosx64-devedition-nightly/opt
-   - win32-devedition-nightly/opt
-   - win64-devedition-nightly/opt
-   - linux64-asan-reporter-nightly/opt
-   - win64-asan-reporter-nightly/opt
+    - linux-nightly/opt
+    - linux64-nightly/opt
+    - macosx64-nightly/opt
+    - win32-nightly/opt
+    - win64-nightly/opt
+    - linux-devedition-nightly/opt
+    - linux64-devedition-nightly/opt
+    - macosx64-devedition-nightly/opt
+    - win32-devedition-nightly/opt
+    - win64-devedition-nightly/opt
+    - linux64-asan-reporter-nightly/opt
+    - win64-asan-reporter-nightly/opt
 
 job-template:
-   shipping-phase: promote
+    shipping-phase: promote
--- a/taskcluster/taskgraph/loader/multi_dep.py
+++ b/taskcluster/taskgraph/loader/multi_dep.py
@@ -93,16 +93,62 @@ def platform_grouping(config, tasks):
                 continue
             elif not_platforms and combined_platform in not_platforms:
                 continue
 
         groups.setdefault((platform, build_type, product), []).append(task)
     return groups
 
 
+@group_by('single-locale')
+def single_locale_grouping(config, tasks):
+    """Split by a single locale (but also by platform, build-type, product)
+
+    The locale can be `None` (en-US build/signing/repackage), a single locale,
+    or multiple locales per task, e.g. for l10n chunking. In the case of a task
+    with, say, five locales, the task will show up in all five locale groupings.
+
+    This grouping is written for non-partner-repack beetmover, but might also
+    be useful elsewhere.
+
+    """
+    only_platforms = config.get('only-for-build-platforms')
+    not_platforms = config.get('not-for-build-platforms')
+    groups = {}
+
+    for task in tasks:
+        if task.kind not in config.get('kind-dependencies', []):
+            continue
+        platform = task.attributes.get('build_platform')
+        build_type = task.attributes.get('build_type')
+        product = task.attributes.get('shipping_product',
+                                      task.task.get('shipping-product'))
+        task_locale = task.attributes.get('locale')
+        chunk_locales = task.attributes.get('chunk_locales')
+        locales = chunk_locales or [task_locale]
+
+        # Skip only_ and not_ platforms that don't match
+        if only_platforms or not_platforms:
+            if not platform or not build_type:
+                continue
+            combined_platform = "{}/{}".format(platform, build_type)
+            if only_platforms and combined_platform not in only_platforms:
+                continue
+            elif not_platforms and combined_platform in not_platforms:
+                continue
+
+        for locale in locales:
+            locale_key = (platform, build_type, product, locale)
+            groups.setdefault(locale_key, [])
+            if task not in groups[locale_key]:
+                groups[locale_key].append(task)
+
+    return groups
+
+
 def assert_unique_members(kinds, error_msg=None):
     if len(kinds) != len(set(kinds)):
         raise Exception(error_msg)
 
 
 def get_primary_dep(config, dep_tasks):
     """Find the dependent task to inherit attributes from.
 
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -147,53 +147,55 @@ task_description_schema = {str(k): v for
 transforms = TransformSequence()
 
 # shortcut for a string where task references are allowed
 taskref_or_string = Any(
     basestring,
     {Required('task-reference'): basestring})
 
 beetmover_description_schema = Schema({
-    # the dependent task (object) for this beetmover job, used to inform beetmover.
-    Required('dependent-task'): object,
+    # dictionary of dependent task objects, keyed by kind.
+    Required('dependent-tasks'): {basestring: object},
 
-    Required('grandparent-tasks'): object,
+    # The primary dependency; we key our task info off this dep task
+    Required('primary-dependency'): object,
 
     # depname is used in taskref's to identify the taskID of the unsigned things
     Required('depname', default='build'): basestring,
 
     # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
-    Optional('label'): basestring,
+    Required('label'): basestring,
 
     # treeherder is allowed here to override any defaults we use for beetmover.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
     # below transforms for defaults of various values.
     Optional('treeherder'): task_description_schema['treeherder'],
 
     # locale is passed only for l10n beetmoving
     Optional('locale'): basestring,
     Required('shipping-phase'): task_description_schema['shipping-phase'],
+    # Optional until we fix asan (run_on_projects?)
     Optional('shipping-product'): task_description_schema['shipping-product'],
 })
 
 
 @transforms.add
 def validate(config, jobs):
     for job in jobs:
-        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        label = job.get('primary-dependency', object).__dict__.get('label', '?no-label?')
         validate_schema(
             beetmover_description_schema, job,
             "In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
         yield job
 
 
 @transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
-        dep_job = job['dependent-task']
+        dep_job = job['primary-dependency']
         attributes = dep_job.attributes
 
         treeherder = job.get('treeherder', {})
         treeherder.setdefault('symbol', 'BM-R')
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
         treeherder.setdefault('platform',
                               "{}/opt".format(dep_th_platform))
@@ -204,40 +206,36 @@ def make_task_description(config, jobs):
             "Beetmover submission for locale '{locale}' for build '"
             "{build_platform}/{build_type}'".format(
                 locale=attributes.get('locale', 'en-US'),
                 build_platform=attributes.get('build_platform'),
                 build_type=attributes.get('build_type')
             )
         )
 
-        dependent_kind = dep_job.kind
-        if dependent_kind == 'repackage-signing-l10n':
-            dependent_kind = "repackage-signing"
-        dependencies = {dependent_kind: dep_job}
+        upstream_deps = job['dependent-tasks']
 
+        # TODO fix the upstreamArtifact generation to not need this?
         signing_name = "build-signing"
+        build_name = "build"
+        repackage_name = "repackage"
+        repackage_signing_name = "repackage-signing"
         if job.get('locale'):
             signing_name = "nightly-l10n-signing"
-        dependencies['signing'] = job['grandparent-tasks'][signing_name]
-
-        build_name = "build"
-        if job.get('locale'):
-            build_name = "unsigned-repack"
-        dependencies["build"] = job['grandparent-tasks'][build_name]
-
-        # repackage-l10n actually uses the repackage depname here
-        dependencies["repackage"] = job['grandparent-tasks']["repackage"]
-
-        # If this isn't a direct dependency, it won't be in there.
-        if 'repackage-signing' not in dependencies:
-            repackage_signing_name = "repackage-signing"
-            if job.get('locale'):
-                repackage_signing_name = "repackage-signing-l10n"
-            dependencies["repackage-signing"] = job['grandparent-tasks'][repackage_signing_name]
+            build_name = "nightly-l10n"
+            repackage_name = "repackage-l10n"
+            repackage_signing_name = "repackage-signing-l10n"
+        dependencies = {
+            "build": upstream_deps[build_name],
+            "repackage": upstream_deps[repackage_name],
+            "repackage-signing": upstream_deps[repackage_signing_name],
+            "signing": upstream_deps[signing_name],
+        }
+        if 'partials-signing' in upstream_deps:
+            dependencies['partials-signing'] = upstream_deps['partials-signing']
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         if job.get('locale'):
             attributes['locale'] = job['locale']
 
         bucket_scope = get_beetmover_bucket_scope(config)
         action_scope = get_beetmover_action_scope(config)
 
@@ -348,37 +346,19 @@ def _check_platform_matched_only_one_reg
 least 2 regular expressions. First matched: "{first_matched}". Second matched: \
 "{second_matched}"'.format(
             task_type=task_type, platform=platform,
             first_matched=platform_was_previously_matched_by_regex.pattern,
             second_matched=platform_regex.pattern
         ))
 
 
-def is_valid_beetmover_job(job):
-    # beetmover after partials-signing should have six dependencies.
-    # windows builds w/o partials don't have docker-image, so fewer
-    # dependencies
-    if 'partials-signing' in job['dependencies'].keys():
-        expected_dep_count = 5
-    else:
-        expected_dep_count = 4
-
-    return (len(job["dependencies"]) == expected_dep_count and
-            any(['repackage' in j for j in job['dependencies']]))
-
-
 @transforms.add
 def make_task_worker(config, jobs):
     for job in jobs:
-        if not is_valid_beetmover_job(job):
-            raise NotImplementedError(
-                "{}: Beetmover_repackage must have five dependencies.".format(job['label'])
-            )
-
         locale = job["attributes"].get("locale")
         platform = job["attributes"]["build_platform"]
 
         upstream_artifacts = generate_upstream_artifacts(
             job, job['dependencies'], platform, locale,
             project=config.params['project']
         )
 
@@ -401,34 +381,25 @@ def make_partials_artifacts(config, jobs
         if not locale:
             locale = 'en-US'
 
         # Remove when proved reliable
         # job['treeherder']['tier'] = 3
 
         platform = job["attributes"]["build_platform"]
 
+        if 'partials-signing' not in job['dependencies']:
+            logger.debug("beetmover-repackage partials finished, no partials")
+            yield job
+            continue
+
         balrog_platform = get_balrog_platform_name(platform)
-
         artifacts = get_partials_artifacts(config.params.get('release_history'),
                                            balrog_platform, locale)
 
-        # Dependency:        | repackage-signing | partials-signing
-        # Partials artifacts |              Skip | Populate & yield
-        # No partials        |             Yield |         continue
-        if len(artifacts) == 0:
-            if 'partials-signing' in job['dependencies']:
-                continue
-            else:
-                yield job
-                continue
-        else:
-            if 'partials-signing' not in job['dependencies']:
-                continue
-
         upstream_artifacts = generate_partials_upstream_artifacts(
             job, artifacts, balrog_platform, locale
         )
 
         job['worker']['upstream-artifacts'].extend(upstream_artifacts)
 
         extra = list()
 
--- a/taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py
@@ -11,17 +11,17 @@ from taskgraph.transforms.base import Tr
 from taskgraph.util.treeherder import join_symbol
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def make_beetmover_description(config, jobs):
     for job in jobs:
-        dep_job = job['dependent-task']
+        dep_job = job['primary-dependency']
 
         locale = dep_job.attributes.get('locale')
         if not locale:
             yield job
             continue
 
         group = 'BMR-L10n'
 
@@ -29,16 +29,16 @@ def make_beetmover_description(config, j
         symbol = locale
 
         treeherder = {
             'symbol': join_symbol(group, symbol),
         }
 
         beet_description = {
             'label': job['label'],
-            'dependent-task': dep_job,
-            'grandparent-tasks': job['grandparent-tasks'],
+            'primary-dependency': dep_job,
+            'dependent-tasks': job['dependent-tasks'],
             'treeherder': treeherder,
             'locale': locale,
-            'shipping-phase': job.get('shipping-phase'),
-            'shipping-product': job.get('shipping-product'),
+            'shipping-phase': job['shipping-phase'],
+            'shipping-product': job['shipping-product'],
         }
         yield beet_description