Bug 1469436 - Create a multi_dep loader and assign a first consumer in l10n. r=tomprince r=aki
authorJustin Wood <Callek@gmail.com>
Fri, 15 Jun 2018 11:50:50 -0700
changeset 474615 33f3e7a6ca7a5e1c7d0a9fc0aaeb3bde5493facd
parent 474614 2477607822a3c5f7ce1f76220910ccbed4516fb1
child 474677 9cb2150c941b225cf14af83ec18613d43ff32868
push id204
push userfmarier@mozilla.com
push dateWed, 25 Jul 2018 00:48:09 +0000
reviewerstomprince, aki
bugs1469436
milestone63.0a1
Bug 1469436 - Create a multi_dep loader and assign a first consumer in l10n. r=tomprince r=aki Differential Revision: https://phabricator.services.mozilla.com/D1696
taskcluster/ci/l10n/kind.yml
taskcluster/ci/nightly-l10n/kind.yml
taskcluster/taskgraph/loader/multi_dep.py
taskcluster/taskgraph/transforms/l10n.py
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -1,23 +1,27 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-loader: taskgraph.loader.single_dep:loader
+loader: taskgraph.loader.multi_dep:loader
 
+group-by: platform
 
 transforms:
    - taskgraph.transforms.l10n:transforms
    - taskgraph.transforms.use_toolchains:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - build
+   - build-signing
+   - repackage
+   - repackage-signing
    - toolchain
 
 only-for-build-platforms:
    - linux64/opt
    - linux/opt
    - android-api-16/opt
    - macosx64/opt
    - win32/opt
@@ -94,37 +98,37 @@ job-template:
             win32: windows2012-32/opt
             win64: windows2012-64/opt
             android-api-16: android-4-0-armv7-api16/opt
    env:
       by-build-platform:
          linux.*:   # linux64 and 32 get same treatment here
             EN_US_PACKAGE_NAME: target.tar.bz2
             EN_US_BINARY_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<signed-build>/artifacts/{artifact_prefix}
+               task-reference: https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/{artifact_prefix}
             MAR_TOOLS_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
+               task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
          macosx64:
             EN_US_PACKAGE_NAME: target.dmg
             EN_US_BINARY_URL:
                task-reference: https://queue.taskcluster.net/v1/task/<repackage>/artifacts/{artifact_prefix}
             MAR_TOOLS_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
+               task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
          win.*:
             EN_US_PACKAGE_NAME: target.zip
             EN_US_BINARY_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<signed-build>/artifacts/{artifact_prefix}
+               task-reference: https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/{artifact_prefix}
             EN_US_INSTALLER_BINARY_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<repackage-signed>/artifacts/{artifact_prefix}
+               task-reference: https://queue.taskcluster.net/v1/task/<repackage-signing>/artifacts/{artifact_prefix}
             MAR_TOOLS_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
+               task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
          android-api-16:
             EN_US_PACKAGE_NAME: target.apk
             EN_US_BINARY_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}
+               task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}
    mozharness:
       config:
          by-build-platform:
             linux:
                - single_locale/{project}.py
                - single_locale/linux32.py
                - single_locale/tc_common.py
                - single_locale/tc_linux32.py
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -1,22 +1,27 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-loader: taskgraph.loader.single_dep:loader
+loader: taskgraph.loader.multi_dep:loader
+
+group-by: platform
 
 transforms:
    - taskgraph.transforms.l10n:transforms
    - taskgraph.transforms.use_toolchains:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - build
+   - build-signing
+   - repackage
+   - repackage-signing
    - toolchain
 
 only-for-build-platforms:
    - linux64-nightly/opt
    - linux-nightly/opt
    - android-api-16-nightly/opt
    - macosx64-nightly/opt
    - win32-nightly/opt
@@ -118,37 +123,37 @@ job-template:
             win32-devedition-nightly: windows2012-32-devedition/opt
             win64-devedition-nightly: windows2012-64-devedition/opt
             android-api-16-nightly: android-4-0-armv7-api16/opt
    env:
       by-build-platform:
          linux.*:   # linux64 and 32 get same treatment here
             EN_US_PACKAGE_NAME: target.tar.bz2
             EN_US_BINARY_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<signed-build>/artifacts/{artifact_prefix}
+               task-reference: https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/{artifact_prefix}
             MAR_TOOLS_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
+               task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
          macosx64.*:
             EN_US_PACKAGE_NAME: target.dmg
             EN_US_BINARY_URL:
                task-reference: https://queue.taskcluster.net/v1/task/<repackage>/artifacts/{artifact_prefix}
             MAR_TOOLS_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
+               task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
          win.*:
             EN_US_PACKAGE_NAME: target.zip
             EN_US_BINARY_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<signed-build>/artifacts/{artifact_prefix}
+               task-reference: https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/{artifact_prefix}
             EN_US_INSTALLER_BINARY_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<repackage-signed>/artifacts/{artifact_prefix}
+               task-reference: https://queue.taskcluster.net/v1/task/<repackage-signing>/artifacts/{artifact_prefix}
             MAR_TOOLS_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/host/bin
+               task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/host/bin
          android-api-16-nightly:
             EN_US_PACKAGE_NAME: target.apk
             EN_US_BINARY_URL:
-               task-reference: https://queue.taskcluster.net/v1/task/<unsigned-build>/artifacts/{artifact_prefix}/en-US
+               task-reference: https://queue.taskcluster.net/v1/task/<build>/artifacts/{artifact_prefix}/en-US
    mozharness:
       config:
          by-build-platform:
             linux-nightly:
                - single_locale/{project}.py
                - single_locale/linux32.py
                - single_locale/tc_common.py
                - single_locale/tc_linux32.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/loader/multi_dep.py
@@ -0,0 +1,99 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+
+
+# Define a collection of group_by functions
+GROUP_BY_MAP = {}
+
+
+def group_by(name):
+    def wrapper(func):
+        GROUP_BY_MAP[name] = func
+        return func
+    return wrapper
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    """
+    Load tasks based on the jobs dependant kinds, designed for use as
+    multiple-dependent needs.
+
+    Required ``group-by-fn`` is used to define how we coalesce the
+    multiple deps together to pass to transforms, e.g. all kinds specified get
+    collapsed by platform with `platform`
+
+    The `only-for-build-platforms` kind configuration, if specified, will limit
+    the build platforms for which a job will be created. Alternatively there is
+    'not-for-build-platforms' kind configuration which will be consulted only after
+    'only-for-build-platforms' is checked (if present), and omit any jobs where the
+    build platform matches.
+
+    Optional ``job-template`` kind configuration value, if specified, will be used to
+    pass configuration down to the specified transforms used.
+    """
+    job_template = config.get('job-template')
+
+    for dep_tasks in group_tasks(config, loaded_tasks):
+        job = {'dependent-tasks': dep_tasks}
+        if job_template:
+            job.update(copy.deepcopy(job_template))
+        # copy shipping_product from upstream
+        product = dep_tasks[dep_tasks.keys()[0]].attributes.get(
+            'shipping_product',
+            dep_tasks[dep_tasks.keys()[0]].task.get('shipping-product')
+        )
+        if product:
+            job.setdefault('shipping-product', product)
+
+        yield job
+
+
+def group_tasks(config, tasks):
+    group_by_fn = GROUP_BY_MAP[config['group-by']]
+
+    groups = group_by_fn(config, tasks)
+
+    for combinations in groups.itervalues():
+        kinds = [f.kind for f in combinations]
+        assert_unique_members(kinds, error_msg=(
+            "Multi_dep.py should have filtered down to one task per kind"))
+        dependencies = {t.kind: copy.deepcopy(t) for t in combinations}
+        yield dependencies
+
+
+@group_by('platform')
+def platform_grouping(config, tasks):
+    only_platforms = config.get('only-for-build-platforms')
+    not_platforms = config.get('not-for-build-platforms')
+
+    groups = {}
+    for task in tasks:
+        if task.kind not in config.get('kind-dependencies', []):
+            continue
+        platform = task.attributes.get('build_platform')
+        build_type = task.attributes.get('build_type')
+        product = task.attributes.get('shipping_product',
+                                      task.task.get('shipping-product'))
+
+        # Skip only_ and not_ platforms that don't match
+        if only_platforms or not_platforms:
+            if not platform or not build_type:
+                continue
+            combined_platform = "{}/{}".format(platform, build_type)
+            if only_platforms and combined_platform not in only_platforms:
+                continue
+            elif not_platforms and combined_platform in not_platforms:
+                continue
+
+        groups.setdefault((platform, build_type, product), []).append(task)
+    return groups
+
+
+def assert_unique_members(kinds, error_msg=None):
+    if len(kinds) != len(set(kinds)):
+        raise Exception(error_msg)
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -92,18 +92,18 @@ l10n_description_schema = Schema({
         # Type of index
         Optional('type'): basestring,
     },
     # Description of the localized task
     Required('description'): _by_platform(basestring),
 
     Optional('run-on-projects'): job_description_schema['run-on-projects'],
 
-    # task object of the dependent task
-    Required('dependent-task'): object,
+    # dictionary of dependent task objects, keyed by kind.
+    Required('dependent-tasks'): {basestring: object},
 
     # worker-type to utilize
     Required('worker-type'): _by_platform(basestring),
 
     # File which contains the used locales
     Required('locales-file'): _by_platform(basestring),
 
     # Tooltool visibility required for task.
@@ -192,28 +192,28 @@ def _remove_locales(locales, to_remove=N
     return {
         locale: revision for locale, revision in locales.items() if locale not in to_remove
     }
 
 
 @transforms.add
 def setup_name(config, jobs):
     for job in jobs:
-        dep = job['dependent-task']
+        dep = job['dependent-tasks']['build']
         # Set the name to the same as the dep task, without kind name.
         # Label will get set automatically with this kinds name.
         job['name'] = job.get('name',
                               dep.task['metadata']['name'][len(dep.kind) + 1:])
         yield job
 
 
 @transforms.add
 def copy_in_useful_magic(config, jobs):
     for job in jobs:
-        dep = job['dependent-task']
+        dep = job['dependent-tasks']['build']
         attributes = copy_attributes_from_dependent_job(dep)
         attributes.update(job.get('attributes', {}))
         # build-platform is needed on `job` for by-build-platform
         job['build-platform'] = attributes.get("build_platform")
         job['attributes'] = attributes
         yield job
 
 
@@ -224,30 +224,29 @@ def validate_early(config, jobs):
                         "In job {!r}:".format(job.get('name', 'unknown')))
         yield job
 
 
 @transforms.add
 def setup_nightly_dependency(config, jobs):
     """ Sets up a task dependency to the signing job this relates to """
     for job in jobs:
-        job['dependencies'] = {'unsigned-build': job['dependent-task'].label}
+        job['dependencies'] = {'build': job['dependent-tasks']['build'].label}
         if job['attributes']['build_platform'].startswith('win') or \
                 job['attributes']['build_platform'].startswith('linux'):
-            # Weave these in and just assume they will be there in the resulting graph
             job['dependencies'].update({
-                'signed-build': 'build-signing-{}'.format(job['name']),
+                'build-signing': job['dependent-tasks']['build-signing'].label,
             })
         if job['attributes']['build_platform'].startswith('macosx'):
             job['dependencies'].update({
-                'repackage': 'repackage-{}'.format(job['name'])
+                'repackage': job['dependent-tasks']['repackage'].label
             })
         if job['attributes']['build_platform'].startswith('win'):
             job['dependencies'].update({
-                'repackage-signed': 'repackage-signing-{}'.format(job['name'])
+                'repackage-signing': job['dependent-tasks']['repackage-signing'].label
             })
         yield job
 
 
 @transforms.add
 def handle_keyed_by(config, jobs):
     """Resolve fields that can be keyed by platform, etc."""
     fields = [