Bug 1482395: [mozrelease] Generate whats new blob and populate it during release; r=mtabara
authorTom Prince <mozilla@hocat.ca>
Mon, 26 Nov 2018 17:51:23 +0000
changeset 507562 3bf6a4a7e135ef3df9ac432097fea323d7b86668
parent 507561 b09a44bf41cb77963e1adbd15adfbd3996dcabb2
child 507563 7a0d7f5cae42b0c3398423c4da3afe03ad9b0946
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara
bugs1482395
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1482395: [mozrelease] Generate whats new blob and populate it during release; r=mtabara Differential Revision: https://phabricator.services.mozilla.com/D12708
browser/config/whats_new_page.yml
build/sparse-profiles/taskgraph
taskcluster/ci/balrog/kind.yml
taskcluster/ci/release-balrog-submit-toplevel/kind.yml
taskcluster/ci/release-secondary-balrog-submit-toplevel/kind.yml
taskcluster/taskgraph/transforms/balrog_submit.py
taskcluster/taskgraph/transforms/balrog_toplevel.py
taskcluster/taskgraph/transforms/task.py
tools/lint/yaml.yml
new file mode 100644
--- /dev/null
+++ b/browser/config/whats_new_page.yml
@@ -0,0 +1,57 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+---
+- type: product-details
+  # %LOCALE% is automatically replaced by Balrog.
+  url: "https://www.mozilla.org/%LOCALE%/{product}/{version}/releasenotes/"
+- type: show-url
+  # yamllint disable-line rule:line-length
+  url: "https://www.mozilla.org/%LOCALE%/{product}/{version}/whatsnew/?oldversion=%OLD_VERSION%"
+  conditions:
+      blob-types: [wnp]
+      release-types: [release, release-rc]
+      products: [firefox]
+      update-channel: release
+      # e.g.: ["<61.0"]. {version.major_number} reflects the current version.
+      # This is done by taskgraph.
+      versions: ["<{version.major_number}.0"]
+      locales:
+          - az
+          - be
+          - cak
+          - cy
+          - da
+          - de
+          - dsb
+          - en-CA
+          - en-GB
+          - en-US
+          - es-AR
+          - et
+          - fr
+          - hi-IN
+          - hsb
+          - hu
+          - ia
+          - id
+          - it
+          - ka
+          - lij
+          - lt
+          - ms
+          - nb-NO
+          - nl
+          - pl
+          - pt-PT
+          - ro
+          - ru
+          - sk
+          - sl
+          - sq
+          - sv-SE
+          - th
+          - tr
+          - zh-CN
+          - zh-TW
--- a/build/sparse-profiles/taskgraph
+++ b/build/sparse-profiles/taskgraph
@@ -4,16 +4,17 @@
 # These files are read as part of generating the taskgraph.
 path:browser/locales/l10n-changesets.json
 path:browser/locales/l10n-onchange-changesets.json
 path:mobile/locales/l10n-changesets.json
 path:mobile/locales/l10n-onchange-changesets.json
 path:browser/locales/shipped-locales
 path:browser/config/version_display.txt
 path:browser/config/version.txt
+path:browser/config/whats_new_page.yml
 
 # Lots of random files in here are read. Just pull in the whole thing.
 path:build/
 
 # TODO remove once bug 1402010 is resolved and test manifests aren't
 # processed in Files() reading mode in moz.build files.
 path:layout/tools/reftest/
 path:testing/web-platform/tests/tools/
--- a/taskcluster/ci/balrog/kind.yml
+++ b/taskcluster/ci/balrog/kind.yml
@@ -1,20 +1,26 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
-   - taskgraph.transforms.name_sanity:transforms
-   - taskgraph.transforms.balrog_submit:transforms
-   - taskgraph.transforms.scriptworker:add_balrog_scopes
-   - taskgraph.transforms.task:transforms
+    - taskgraph.transforms.name_sanity:transforms
+    - taskgraph.transforms.balrog_submit:transforms
+    - taskgraph.transforms.scriptworker:add_balrog_scopes
+    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-   - beetmover
-   - beetmover-l10n
-   - beetmover-repackage
+    - beetmover
+    - beetmover-l10n
+    - beetmover-repackage
 
 only-for-attributes:
-   - nightly
-   - signed
+    - nightly
+    - signed
+
+job-template:
+    update-no-wnp:
+        by-release-type:
+            nightly: false
+            default: true
--- a/taskcluster/ci/release-balrog-submit-toplevel/kind.yml
+++ b/taskcluster/ci/release-balrog-submit-toplevel/kind.yml
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
    - taskgraph.transforms.release:run_on_releases
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.scriptworker:add_balrog_scopes
+   - taskgraph.transforms.balrog_toplevel:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
    run-on-projects: []  # to make sure this never runs in CI
    shipping-phase: promote
    worker:
       implementation: balrog
       balrog-action: submit-toplevel
@@ -20,16 +21,17 @@ job-defaults:
       archive-domain:
          by-release-level:
             production: archive.mozilla.org
             staging: ftp.stage.mozaws.net
       download-domain:
          by-release-level:
             production: download.mozilla.org
             staging: bouncer-bouncer-releng.stage.mozaws.net
+   whats-new-config: browser/config/whats_new_page.yml
 
 jobs:
    firefox:
       name: submit-toplevel-firefox-release-to-balrog
       description: Submit toplevel Firefox release to balrog
       shipping-product: firefox
       worker:
          product: firefox
--- a/taskcluster/ci/release-secondary-balrog-submit-toplevel/kind.yml
+++ b/taskcluster/ci/release-secondary-balrog-submit-toplevel/kind.yml
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
    - taskgraph.transforms.release_deps:transforms
    - taskgraph.transforms.scriptworker:add_balrog_scopes
+   - taskgraph.transforms.balrog_toplevel:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
    run-on-projects: []  # to make sure this never runs in CI
    shipping-phase: promote
    worker:
       implementation: balrog
       balrog-action: submit-toplevel
@@ -20,16 +21,17 @@ job-defaults:
       archive-domain:
          by-release-level:
             production: archive.mozilla.org
             staging: ftp.stage.mozaws.net
       download-domain:
          by-release-level:
             production: download.mozilla.org
             staging: bouncer-bouncer-releng.stage.mozaws.net
+   whats-new-config: browser/config/whats_new_page.yml
 
 jobs:
    firefox:
       name: submit-toplevel-rc-firefox-release-to-balrog
       description: submit toplevel RC Firefox release to balrog
       shipping-product: firefox
       worker:
          product: firefox
--- a/taskcluster/taskgraph/transforms/balrog_submit.py
+++ b/taskcluster/taskgraph/transforms/balrog_submit.py
@@ -5,16 +5,19 @@
 Transform the per-locale balrog task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.schema import (
+    optionally_keyed_by, resolve_keyed_by,
+)
 from taskgraph.util.scriptworker import (
     get_balrog_server_scope, get_worker_type_for_scope
 )
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
@@ -25,16 +28,22 @@ task_description_schema = {str(k): v for
 taskref_or_string = Any(
     basestring,
     {Required('task-reference'): basestring})
 
 balrog_description_schema = schema.extend({
     # unique label to describe this balrog task, defaults to balrog-{dep.label}
     Optional('label'): basestring,
 
+
+    Optional(
+        'update-no-wnp',
+        description="Whether the parallel `-No-WNP` blob should be updated as well.",
+    ): optionally_keyed_by('release-type', bool),
+
     # treeherder is allowed here to override any defaults we use for beetmover.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
     # below transforms for defaults of various values.
     Optional('treeherder'): task_description_schema['treeherder'],
 
     Optional('attributes'): task_description_schema['attributes'],
 
     # Shipping product / phase
@@ -43,16 +52,35 @@ balrog_description_schema = schema.exten
 })
 
 
 transforms = TransformSequence()
 transforms.add_validate(balrog_description_schema)
 
 
 @transforms.add
+def handle_keyed_by(config, jobs):
+    """Resolve fields that can be keyed by platform, etc."""
+    fields = [
+        "update-no-wnp",
+    ]
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        for field in fields:
+            resolve_keyed_by(
+                item=job, field=field, item_name=label,
+                **{
+                    'project': config.params['project'],
+                    'release-type': config.params['release_type'],
+                }
+            )
+        yield job
+
+
+@transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
         dep_job = job['primary-dependency']
 
         treeherder = job.get('treeherder', {})
         treeherder.setdefault('symbol', 'c-Up(N)')
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
@@ -96,16 +124,17 @@ def make_task_description(config, jobs):
         task = {
             'label': label,
             'description': description,
             'worker-type': get_worker_type_for_scope(config, server_scope),
             'worker': {
                 'implementation': 'balrog',
                 'upstream-artifacts': upstream_artifacts,
                 'balrog-action': 'submit-locale',
+                'suffixes': ['', '-No-WNP'] if job.get('update-no-wnp') else [''],
             },
             'dependencies': {'beetmover': dep_job.label},
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
             'shipping-phase': job.get('shipping-phase', 'promote'),
             'shipping-product': job.get('shipping-product'),
         }
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/balrog_toplevel.py
@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import yaml
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.scriptworker import (
+    get_release_config,
+)
+
+from mozrelease.balrog import generate_update_properties
+from mozilla_version.gecko import GeckoVersion
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def generate_update_line(config, jobs):
+    """Resolve fields that can be keyed by platform, etc."""
+    release_config = get_release_config(config)
+    for job in jobs:
+        config_file = job.pop('whats-new-config')
+        with open(config_file, "rb") as f:
+            update_config = yaml.safe_load(f)
+
+        job['worker']['update-line'] = {}
+        for blob_type, suffix in [('wnp', ''), ('no-wnp', '-No-WNP')]:
+            context = {
+                'release-type': config.params['release_type'],
+                'product': job['shipping-product'],
+                'version': GeckoVersion.parse(release_config['version']),
+                'blob-type': blob_type,
+            }
+            job['worker']['update-line'][suffix] = generate_update_properties(
+                context, update_config)
+
+        yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -1003,16 +1003,19 @@ def build_beetmover_maven_payload(config
     Optional('publish-rules'): optionally_keyed_by('release-type', 'release-level', [int]),
     Optional('rules-to-update'): optionally_keyed_by(
         'release-type', 'release-level', [basestring]),
     Optional('archive-domain'): optionally_keyed_by('release-level', basestring),
     Optional('download-domain'): optionally_keyed_by('release-level', basestring),
     Optional('blob-suffix'): basestring,
     Optional('complete-mar-filename-pattern'): basestring,
     Optional('complete-mar-bouncer-product-pattern'): basestring,
+    Optional('update-line'): object,
+    Optional('suffixes'): [basestring],
+
 
     # list of artifact URLs for the artifacts that should be beetmoved
     Optional('upstream-artifacts'): [{
         # taskId of the task with the artifact
         Required('taskId'): taskref_or_string,
 
         # type of signing task (for CoT)
         Required('taskType'): basestring,
@@ -1022,17 +1025,18 @@ def build_beetmover_maven_payload(config
     }],
 })
 def build_balrog_payload(config, task, task_def):
     worker = task['worker']
     release_config = get_release_config(config)
 
     if worker['balrog-action'] == 'submit-locale':
         task_def['payload'] = {
-            'upstreamArtifacts':  worker['upstream-artifacts']
+            'upstreamArtifacts':  worker['upstream-artifacts'],
+            'suffixes': worker['suffixes'],
         }
     else:
         for prop in ('archive-domain', 'channel-names', 'download-domain',
                      'publish-rules', 'rules-to-update'):
             if prop in worker:
                 resolve_keyed_by(
                     worker, prop, task['description'],
                     **{
@@ -1054,16 +1058,17 @@ def build_balrog_payload(config, task, t
                 'app_version': release_config['appVersion'],
                 'archive_domain': worker['archive-domain'],
                 'channel_names': worker['channel-names'],
                 'download_domain': worker['download-domain'],
                 'partial_versions': release_config.get('partial_versions', ""),
                 'platforms': worker['platforms'],
                 'rules_to_update': worker['rules-to-update'],
                 'require_mirrors': worker['require-mirrors'],
+                'update_line': worker['update-line'],
             })
         else:  # schedule / ship
             task_def['payload'].update({
                 'publish_rules': worker['publish-rules'],
                 'release_eta': worker.get('release-eta', config.params.get('release_eta')) or '',
             })
 
 
--- a/tools/lint/yaml.yml
+++ b/tools/lint/yaml.yml
@@ -1,13 +1,14 @@
 ---
 yamllint:
     description: YAML linter
     include:
         - .cron.yml
+        - browser/config/
         - python/mozlint/
         - security/nss/.taskcluster.yml
         - taskcluster
         - testing/mozharness
         - tools
     extensions: ['yml', 'yaml']
     support-files:
         - '**/.yamllint'