Bug 1481916 - add l10n-bumper task. r=Callek
authorAki Sasaki <asasaki@mozilla.com>
Mon, 18 Nov 2019 18:21:58 +0000
changeset 502514 11f48aaae9552c1f8da22578aedc41d5fa07ebbc
parent 502513 fdd07df83c87f12725f4b97c80e644fd11673977
child 502515 ee452cb16fac66c1f5011cef8435f7e2b3e83502
push id114172
push userdluca@mozilla.com
push dateTue, 19 Nov 2019 11:31:10 +0000
treeherdermozilla-inbound@b5c5ba07d3db [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek
bugs1481916
milestone72.0a1
Bug 1481916 - add l10n-bumper task. r=Callek Differential Revision: https://phabricator.services.mozilla.com/D53116
.cron.yml
taskcluster/ci/config.yml
taskcluster/ci/l10n-bump/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/release_version_bump.py
taskcluster/taskgraph/transforms/task.py
--- a/.cron.yml
+++ b/.cron.yml
@@ -213,8 +213,20 @@ jobs:
       job:
           type: decision-task
           treeherder-symbol: condprof
           target-tasks-method: condprof
       run-on-projects:
           - mozilla-central
       when:
           - {hour: 16, minute: 00}
+
+    - name: l10n-bumper
+      job:
+          type: decision-task
+          treeherder-symbol: l10n-bump
+          target-tasks-method: l10n_bump
+      run-on-projects:
+          - mozilla-central
+          - mozilla-beta
+          - mozilla-esr68
+      when:
+          - {minute: 45}
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -29,16 +29,17 @@ treeherder:
         'Btime-fennec-1proc': 'Browsertime performance tests on Fennec without e10s'
         'Btime-refbrow': 'Browsertime performance tests on the reference browser'
         'Fetch': 'Fetch and store content'
         'Fxfn-l': 'Firefox functional tests (local)'
         'Fxfn-l-1proc': 'Firefox functional tests (local) without e10s'
         'Fxfn-r': 'Firefox functional tests (remote)'
         'Fxfn-r-1proc': 'Firefox functional tests (remote) without e10s'
         'iris': 'Iris testing suite'
+        'l10n-bump': 'L10n Bumper'
         'M': 'Mochitests'
         'M-1proc': 'Mochitests without e10s'
         'M-fis': 'Mochitests with fission enabled'
         'M-spi': 'Mochitests with socket process'
         'M-spi-1proc': 'Mochitests with socket process without e10s'
         'M-sw': 'Mochitests with serviceworker redesign'
         'M-sw-1proc': 'Mochitests with serviceworker redesign without e10s'
         'M-V-1proc': 'Mochitests on Valgrind without e10s'
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/l10n-bump/kind.yml
@@ -0,0 +1,86 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.release_version_bump:transforms
+    - taskgraph.transforms.task:transforms
+
+
+job-defaults:
+    worker-type:
+        by-project:
+            mozilla-(beta|central|release|esr.*): scriptworker-k8s/gecko-3-tree
+            default: scriptworker-k8s/gecko-1-tree
+    worker:
+        implementation: treescript
+        tags: []
+        bump: false
+        dontbuild: true
+        ignore-closed-tree: False
+        push:
+            by-project:
+                mozilla-(central|beta|release|esr.*): true
+                default: false
+        l10n-bump-info:
+            by-release-type:
+                esr.*:
+                    name: Firefox l10n changesets
+                    path: mobile/locales/l10n-changesets.json
+                    version-path: mobile/android/config/version-files/release/version.txt
+                    revision-url: "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(MAJOR_VERSION)s"
+                    platform-configs: [{
+                        "platforms": ["android-multilocale"],
+                        "path": "mobile/android/locales/maemo-locales",
+                    }]
+                default:
+                    name: Firefox l10n changesets
+                    path: browser/locales/l10n-changesets.json
+                    version-path: browser/config/version.txt
+                    revision-url: "https://l10n.mozilla.org/shipping/l10n-changesets?av=fx%(MAJOR_VERSION)s"
+                    ignore-config:
+                        ja:
+                            - macosx64
+                            - macosx64-devedition
+                        ja-JP-mac:
+                            - linux
+                            - linux-devedition
+                            - linux64
+                            - linux64-devedition
+                            - win32
+                            - win32-devedition
+                            - win64
+                            - win64-devedition
+                            - win64-aarch64
+                            - win64-aarch64-devedition
+                    platform-configs: [{
+                        "platforms": [
+                            "linux",
+                            "linux-devedition",
+                            "linux64",
+                            "linux64-devedition",
+                            "macosx64",
+                            "macosx64-devedition",
+                            "win32",
+                            "win32-devedition",
+                            "win64",
+                            "win64-devedition",
+                            "win64-aarch64",
+                            "win64-aarch64-devedition",
+                        ],
+                        "path": "browser/locales/shipped-locales",
+                        "format": "shipped-locales",
+                    }]
+
+jobs:
+    l10n-bumper:
+        name: l10n_bumper
+        description: l10n changesets bumper
+        run-on-projects: []
+        treeherder:
+            kind: build
+            platform: linux64/opt
+            symbol: l10n-bump
+            tier: 1
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -611,8 +611,12 @@ by other tasks.
 visual-metrics-dep
 ------------------
 Tasks that compute visual performance metrics from videos and images captured
 by another task that produces a jobs.json artifact
 
 iris
 ----
 Iris testing suite
+
+l10n-bump
+---------
+Cron-driven tasks that bump l10n-changesets files in-tree, using data from the l10n dashboard.
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -693,16 +693,26 @@ def target_tasks_file_update(full_task_g
     """Select the set of tasks required to perform nightly in-tree file updates
     """
     def filter(task):
         # For now any task in the repo-update kind is ok
         return task.kind in ['repo-update']
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
+@_target_task('l10n_bump')
+def target_tasks_l10n_bump(full_task_graph, parameters, graph_config):
+    """Select the set of tasks required to perform l10n bumping.
+    """
+    def filter(task):
+        # For now any task in the repo-update kind is ok
+        return task.kind in ['l10n-bump']
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+
 @_target_task('cron_bouncer_check')
 def target_tasks_bouncer_check(full_task_graph, parameters, graph_config):
     """Select the set of tasks required to perform bouncer version verification.
     """
     def filter(task):
         if not filter_for_project(task, parameters):
             return False
         # For now any task in the repo-update kind is ok
--- a/taskcluster/taskgraph/transforms/release_version_bump.py
+++ b/taskcluster/taskgraph/transforms/release_version_bump.py
@@ -11,18 +11,21 @@ from taskgraph.transforms.base import Tr
 from taskgraph.util.schema import resolve_keyed_by
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def handle_keyed_by(config, tasks):
     """Resolve fields that can be keyed by platform, etc."""
-    fields = [
+    default_fields = [
         'worker.push',
         'worker.bump-files',
         'worker-type',
     ]
     for task in tasks:
+        fields = default_fields[:]
+        if task["worker"].get("l10n-bump-info"):
+            fields.append('worker.l10n-bump-info')
         for field in fields:
             resolve_keyed_by(task, field, item_name=task['name'],
-                             project=config.params['project'])
+                             **config.params)
         yield task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -1239,25 +1239,39 @@ def build_push_addons_payload(config, ta
 
 
 @payload_builder('treescript', schema={
     Required('tags'): [Any('buildN', 'release', None)],
     Required('bump'): bool,
     Optional('bump-files'): [basestring],
     Optional('repo-param-prefix'): basestring,
     Optional('dontbuild'): bool,
+    Optional('ignore-closed-tree'): bool,
     Required('force-dry-run', default=True): bool,
-    Required('push', default=False): bool
+    Required('push', default=False): bool,
+    Optional('l10n-bump-info'): {
+        Required('name'): basestring,
+        Required('path'): basestring,
+        Required('version-path'): basestring,
+        Required('revision-url'): basestring,
+        Optional('ignore-config'): object,
+        Required('platform-configs'): [{
+            Required('platforms'): [basestring],
+            Required('path'): basestring,
+            Optional('format'): basestring,
+        }],
+    },
 })
 def build_treescript_payload(config, task, task_def):
     worker = task['worker']
     release_config = get_release_config(config)
 
-    task_def['payload'] = {}
+    task_def['payload'] = {'actions': []}
     task_def.setdefault('scopes', [])
+    actions = task_def['payload']['actions']
     if worker['tags']:
         tag_names = []
         product = task['shipping-product'].upper()
         version = release_config['version'].replace('.', '_')
         buildnum = release_config['build_number']
         if 'buildN' in worker['tags']:
             tag_names.extend([
                 "{}_{}_BUILD{}".format(product, version, buildnum),
@@ -1266,37 +1280,48 @@ def build_treescript_payload(config, tas
             tag_names.extend([
               "{}_{}_RELEASE".format(product, version)
             ])
         tag_info = {
             'tags': tag_names,
             'revision': config.params['{}head_rev'.format(worker.get('repo-param-prefix', ''))],
         }
         task_def['payload']['tag_info'] = tag_info
-        task_def['scopes'].append(add_scope_prefix(config, 'treescript:action:tagging'))
+        actions.append('tag')
 
     if worker['bump']:
         if not worker['bump-files']:
             raise Exception("Version Bump requested without bump-files")
 
         bump_info = {}
         bump_info['next_version'] = release_config['next_version']
         bump_info['files'] = worker['bump-files']
         task_def['payload']['version_bump_info'] = bump_info
-        task_def['scopes'].append(add_scope_prefix(config, 'treescript:action:version_bump'))
+        actions.append('version_bump')
+
+    if worker.get('l10n-bump-info'):
+        l10n_bump_info = {}
+        for k, v in worker['l10n-bump-info'].items():
+            l10n_bump_info[k.replace('-', '_')] = worker['l10n-bump-info'][k]
+        task_def['payload']['l10n_bump_info'] = [l10n_bump_info]
+        actions.append('l10n_bump')
 
     if worker['push']:
         task_def['scopes'].append(add_scope_prefix(config, 'treescript:action:push'))
+        actions.append('push')
 
     if worker.get('force-dry-run'):
         task_def['payload']['dry_run'] = True
 
     if worker.get('dontbuild'):
         task_def['payload']['dontbuild'] = True
 
+    if worker.get('ignore-closed-tree') is not None:
+        task_def['payload']['ignore_closed_tree'] = worker['ignore-closed-tree']
+
 
 @payload_builder('invalid', schema={
     # an invalid task is one which should never actually be created; this is used in
     # release automation on branches where the task just doesn't make sense
     Extra: object,
 })
 def build_invalid_payload(config, task, task_def):
     task_def['payload'] = 'invalid task - should never be created'