Bug 1482395: [mozrelease] Generate whats new blob and populate it during release; r=mtabara
authorTom Prince <mozilla@hocat.ca>
Mon, 26 Nov 2018 17:51:23 +0000
changeset 501399 88ab1bde62780758f0234c0ba7704db281397716
parent 501398 e0e9425868e01468879f3cc77a6b8b360b419ded
child 501400 be8fbba20256fdddd75041d7a10a08af088c1daf
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara
bugs1482395
milestone64.0
Bug 1482395: [mozrelease] Generate whats new blob and populate it during release; r=mtabara Differential Revision: https://phabricator.services.mozilla.com/D12708
browser/config/whats_new_page.yml
build/sparse-profiles/taskgraph
taskcluster/ci/balrog/kind.yml
taskcluster/ci/release-balrog-submit-toplevel/kind.yml
taskcluster/ci/release-secondary-balrog-submit-toplevel/kind.yml
taskcluster/taskgraph/transforms/balrog_submit.py
taskcluster/taskgraph/transforms/balrog_toplevel.py
taskcluster/taskgraph/transforms/task.py
tools/lint/yaml.yml
new file mode 100644
--- /dev/null
+++ b/browser/config/whats_new_page.yml
@@ -0,0 +1,57 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+---
+- type: product-details
+  # %LOCALE% is automatically replaced by Balrog.
+  url: "https://www.mozilla.org/%LOCALE%/{product}/{version}/releasenotes/"
+- type: show-url
+  # yamllint disable-line rule:line-length
+  url: "https://www.mozilla.org/%LOCALE%/{product}/{version}/whatsnew/?oldversion=%OLD_VERSION%"
+  conditions:
+      blob-types: [wnp]
+      release-types: [release, release-rc]
+      products: [firefox]
+      update-channel: release
+      # e.g.: ["<61.0"]. {version.major_number} reflects the current version.
+      # This is done by taskgraph.
+      versions: ["<{version.major_number}.0"]
+      locales:
+          - az
+          - be
+          - cak
+          - cy
+          - da
+          - de
+          - dsb
+          - en-CA
+          - en-GB
+          - en-US
+          - es-AR
+          - et
+          - fr
+          - hi-IN
+          - hsb
+          - hu
+          - ia
+          - id
+          - it
+          - ka
+          - lij
+          - lt
+          - ms
+          - nb-NO
+          - nl
+          - pl
+          - pt-PT
+          - ro
+          - ru
+          - sk
+          - sl
+          - sq
+          - sv-SE
+          - th
+          - tr
+          - zh-CN
+          - zh-TW
--- a/build/sparse-profiles/taskgraph
+++ b/build/sparse-profiles/taskgraph
@@ -4,16 +4,17 @@
 # These files are read as part of generating the taskgraph.
 path:browser/locales/l10n-changesets.json
 path:browser/locales/l10n-onchange-changesets.json
 path:mobile/locales/l10n-changesets.json
 path:mobile/locales/l10n-onchange-changesets.json
 path:browser/locales/shipped-locales
 path:browser/config/version_display.txt
 path:browser/config/version.txt
+path:browser/config/whats_new_page.yml
 
 # Lots of random files in here are read. Just pull in the whole thing.
 path:build/
 
 # TODO remove once bug 1402010 is resolved and test manifests aren't
 # processed in Files() reading mode in moz.build files.
 path:layout/tools/reftest/
 path:testing/web-platform/tests/tools/
--- a/taskcluster/ci/balrog/kind.yml
+++ b/taskcluster/ci/balrog/kind.yml
@@ -1,20 +1,26 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
-   - taskgraph.transforms.name_sanity:transforms
-   - taskgraph.transforms.balrog_submit:transforms
-   - taskgraph.transforms.scriptworker:add_balrog_scopes
-   - taskgraph.transforms.task:transforms
+    - taskgraph.transforms.name_sanity:transforms
+    - taskgraph.transforms.balrog_submit:transforms
+    - taskgraph.transforms.scriptworker:add_balrog_scopes
+    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-   - beetmover
-   - beetmover-l10n
-   - beetmover-repackage
+    - beetmover
+    - beetmover-l10n
+    - beetmover-repackage
 
 only-for-attributes:
-   - nightly
-   - signed
+    - nightly
+    - signed
+
+job-template:
+    update-no-wnp:
+        by-release-type:
+            nightly: false
+            default: true
--- a/taskcluster/ci/release-balrog-submit-toplevel/kind.yml
+++ b/taskcluster/ci/release-balrog-submit-toplevel/kind.yml
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
    - taskgraph.transforms.release:run_on_releases
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.scriptworker:add_balrog_scopes
+   - taskgraph.transforms.balrog_toplevel:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
    run-on-projects: []  # to make sure this never runs in CI
    shipping-phase: promote
    worker:
       implementation: balrog
       balrog-action: submit-toplevel
@@ -20,16 +21,17 @@ job-defaults:
       archive-domain:
          by-release-level:
             production: archive.mozilla.org
             staging: ftp.stage.mozaws.net
       download-domain:
          by-release-level:
             production: download.mozilla.org
             staging: bouncer-bouncer-releng.stage.mozaws.net
+   whats-new-config: browser/config/whats_new_page.yml
 
 jobs:
    firefox:
       name: submit-toplevel-firefox-release-to-balrog
       description: Submit toplevel Firefox release to balrog
       shipping-product: firefox
       worker:
          product: firefox
--- a/taskcluster/ci/release-secondary-balrog-submit-toplevel/kind.yml
+++ b/taskcluster/ci/release-secondary-balrog-submit-toplevel/kind.yml
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.scriptworker:add_balrog_scopes
+   - taskgraph.transforms.balrog_toplevel:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
    run-on-projects: []  # to make sure this never runs in CI
    shipping-phase: promote
    worker:
       implementation: balrog
       balrog-action: submit-toplevel
@@ -20,16 +21,17 @@ job-defaults:
       archive-domain:
          by-release-level:
             production: archive.mozilla.org
             staging: ftp.stage.mozaws.net
       download-domain:
          by-release-level:
             production: download.mozilla.org
             staging: bouncer-bouncer-releng.stage.mozaws.net
+   whats-new-config: browser/config/whats_new_page.yml
 
 jobs:
    firefox:
       name: submit-toplevel-rc-firefox-release-to-balrog
       description: submit toplevel RC Firefox release to balrog
       shipping-product: firefox
       worker:
          product: firefox
--- a/taskcluster/taskgraph/transforms/balrog_submit.py
+++ b/taskcluster/taskgraph/transforms/balrog_submit.py
@@ -4,17 +4,20 @@
 """
 Transform the per-locale balrog task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.schema import (
+    validate_schema, Schema,
+    optionally_keyed_by, resolve_keyed_by,
+)
 from taskgraph.util.scriptworker import (
     get_balrog_server_scope, get_worker_type_for_scope
 )
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
@@ -30,16 +33,22 @@ taskref_or_string = Any(
 
 balrog_description_schema = Schema({
     # the dependent task (object) for this balrog job, used to inform balrogworker.
     Required('dependent-task'): object,
 
     # unique label to describe this balrog task, defaults to balrog-{dep.label}
     Optional('label'): basestring,
 
+
+    Optional(
+        'update-no-wnp',
+        description="Whether the parallel `-No-WNP` blob should be updated as well.",
+    ): optionally_keyed_by('release-type', bool),
+
     # treeherder is allowed here to override any defaults we use for beetmover.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
     # below transforms for defaults of various values.
     Optional('treeherder'): task_description_schema['treeherder'],
 
     # Shipping product / phase
     Optional('shipping-product'): task_description_schema['shipping-product'],
     Optional('shipping-phase'): task_description_schema['shipping-phase'],
@@ -53,16 +62,35 @@ def validate(config, jobs):
         validate_schema(
             balrog_description_schema, job,
             "In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
 
         yield job
 
 
 @transforms.add
+def handle_keyed_by(config, jobs):
+    """Resolve fields that can be keyed by platform, etc."""
+    fields = [
+        "update-no-wnp",
+    ]
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        for field in fields:
+            resolve_keyed_by(
+                item=job, field=field, item_name=label,
+                **{
+                    'project': config.params['project'],
+                    'release-type': config.params['release_type'],
+                }
+            )
+        yield job
+
+
+@transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
 
         treeherder = job.get('treeherder', {})
         treeherder.setdefault('symbol', 'c-Up(N)')
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
@@ -106,16 +134,17 @@ def make_task_description(config, jobs):
         task = {
             'label': label,
             'description': description,
             'worker-type': get_worker_type_for_scope(config, server_scope),
             'worker': {
                 'implementation': 'balrog',
                 'upstream-artifacts': upstream_artifacts,
                 'balrog-action': 'submit-locale',
+                'suffixes': ['', '-No-WNP'] if job.get('update-no-wnp') else [''],
             },
             'dependencies': {'beetmover': dep_job.label},
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
             'shipping-phase': job.get('shipping-phase', 'promote'),
             'shipping-product': job.get('shipping-product'),
         }
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/balrog_toplevel.py
@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import yaml
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.scriptworker import (
+    get_release_config,
+)
+
+from mozrelease.balrog import generate_update_properties
+from mozilla_version.gecko import GeckoVersion
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def generate_update_line(config, jobs):
+    """Resolve fields that can be keyed by platform, etc."""
+    release_config = get_release_config(config)
+    for job in jobs:
+        config_file = job.pop('whats-new-config')
+        with open(config_file, "rb") as f:
+            update_config = yaml.safe_load(f)
+
+        job['worker']['update-line'] = {}
+        for blob_type, suffix in [('wnp', ''), ('no-wnp', '-No-WNP')]:
+            context = {
+                'release-type': config.params['release_type'],
+                'product': job['shipping-product'],
+                'version': GeckoVersion.parse(release_config['version']),
+                'blob-type': blob_type,
+            }
+            job['worker']['update-line'][suffix] = generate_update_properties(
+                context, update_config)
+
+        yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -551,16 +551,18 @@ task_description_schema = Schema({
         Optional('publish-rules'): optionally_keyed_by('release-type', 'release-level', [int]),
         Optional('rules-to-update'): optionally_keyed_by(
             'release-type', 'release-level', [basestring]),
         Optional('archive-domain'): optionally_keyed_by('release-level', basestring),
         Optional('download-domain'): optionally_keyed_by('release-level', basestring),
         Optional('blob-suffix'): basestring,
         Optional('complete-mar-filename-pattern'): basestring,
         Optional('complete-mar-bouncer-product-pattern'): basestring,
+        Optional('update-line'): object,
+        Optional('suffixes'): [basestring],
 
         # list of artifact URLs for the artifacts that should be beetmoved
         Optional('upstream-artifacts'): [{
             # taskId of the task with the artifact
             Required('taskId'): taskref_or_string,
 
             # type of signing task (for CoT)
             Required('taskType'): basestring,
@@ -1160,17 +1162,18 @@ def build_beetmover_maven_payload(config
 
 @payload_builder('balrog')
 def build_balrog_payload(config, task, task_def):
     worker = task['worker']
     release_config = get_release_config(config)
 
     if worker['balrog-action'] == 'submit-locale':
         task_def['payload'] = {
-            'upstreamArtifacts':  worker['upstream-artifacts']
+            'upstreamArtifacts':  worker['upstream-artifacts'],
+            'suffixes': worker['suffixes'],
         }
     else:
         for prop in ('archive-domain', 'channel-names', 'download-domain',
                      'publish-rules', 'rules-to-update'):
             if prop in worker:
                 resolve_keyed_by(
                     worker, prop, task['description'],
                     **{
@@ -1192,16 +1195,17 @@ def build_balrog_payload(config, task, t
                 'app_version': release_config['appVersion'],
                 'archive_domain': worker['archive-domain'],
                 'channel_names': worker['channel-names'],
                 'download_domain': worker['download-domain'],
                 'partial_versions': release_config.get('partial_versions', ""),
                 'platforms': worker['platforms'],
                 'rules_to_update': worker['rules-to-update'],
                 'require_mirrors': worker['require-mirrors'],
+                'update_line': worker['update-line'],
             })
         else:  # schedule / ship
             task_def['payload'].update({
                 'publish_rules': worker['publish-rules'],
                 'release_eta': worker.get('release-eta', config.params.get('release_eta')) or '',
             })
 
 
--- a/tools/lint/yaml.yml
+++ b/tools/lint/yaml.yml
@@ -1,13 +1,14 @@
 ---
 yamllint:
     description: YAML linter
     include:
         - .cron.yml
+        - browser/config/
         - python/mozlint/
         - security/nss/.taskcluster.yml
         - taskcluster
         - testing/mozharness
         - tools
     extensions: ['yml', 'yaml']
     support-files:
         - '**/.yamllint'