bug 1423081 - source readme. r=callek
authorAki Sasaki <asasaki@mozilla.com>
Tue, 05 Dec 2017 19:10:01 -0800
changeset 396349 35d738241395d6f4b182151f12ea71313e904900
parent 396348 77de103de179a8169f4d2609280357a63eed44a2
child 396350 960aa6859716029636041811e14a3da379d59d83
push id56975
push userdluca@mozilla.com
push dateThu, 14 Dec 2017 09:59:07 +0000
treeherderautoland@16bcfaad13e1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscallek
bugs1423081, 749312
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1423081 - source readme. r=callek In bug 749312, we were given permission to create a source readme instead of a source tarball. This will save us cycles, disk, and human configuration time. We still need to address the missing balrog_props.json for beetmover-source for that task to turn green. MozReview-Commit-ID: wnyPoNXCsH
taskcluster/ci/beetmover-source/kind.yml
taskcluster/ci/release-source-signing/kind.yml
taskcluster/ci/release-source/kind.yml
taskcluster/ci/release-source/source.yml
taskcluster/taskgraph/loader/build_signing.py
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_source.py
taskcluster/taskgraph/transforms/build_signing.py
taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
taskcluster/taskgraph/util/signed_artifacts.py
testing/mozharness/configs/builds/releng_sub_linux_configs/64_source.py
testing/mozharness/scripts/fx_source.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover-source/kind.yml
@@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.name_sanity:transforms
+   - taskgraph.transforms.beetmover:transforms
+   - taskgraph.transforms.beetmover_source:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - release-source-signing
+
+job-template:
+   shipping-phase: promote
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-source-signing/kind.yml
@@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.build_signing:loader
+
+transforms:
+   - taskgraph.transforms.name_sanity:transforms
+   - taskgraph.transforms.build_signing:transforms
+   - taskgraph.transforms.signing:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - release-source
+
+job-template:
+   shipping-phase: promote
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-source/kind.yml
@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+   - toolchain
+
+transforms:
+   - taskgraph.transforms.build:transforms
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.build_lints:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+jobs-from:
+   - source.yml
+
+job-defaults:
+   shipping-phase: promote
+   notifications:
+      completed:
+         subject: "COMPLETED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         message: "COMPLETED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         plugins:
+            by-project:
+               mozilla-beta: ["log_collect"]
+               mozilla-release: ["log_collect"]
+               default: []
+
+      failed:
+         subject: "FAILED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         message: "FAILED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         plugins:
+            by-project:
+               mozilla-beta: ["log_collect", "ses"]
+               mozilla-release: ["log_collect", "ses"]
+               default: ["ses"]
+         emails:
+            by-project:
+               mozilla-beta: ["release-automation-notifications@mozilla.com"]
+               mozilla-release: ["release-automation-notifications@mozilla.com"]
+               try: ["{task_def[metadata][owner]}"]
+               maple: ["release+tcstaging@mozilla.com"]
+               default: []
+
+      exception:
+         subject: "EXCEPTION: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         message: "EXCEPTION: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
+         plugins:
+            by-project:
+               mozilla-beta: ["log_collect", "ses"]
+               mozilla-release: ["log_collect", "ses"]
+               default: ["ses"]
+         emails:
+            by-project:
+               mozilla-beta: ["release-automation-notifications@mozilla.com"]
+               mozilla-release: ["release-automation-notifications@mozilla.com"]
+               try: ["{task_def[metadata][owner]}"]
+               maple: ["release+tcstaging@mozilla.com"]
+               default: []
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-source/source.yml
@@ -0,0 +1,62 @@
+linux64-source/opt:
+    description: "Firefox source builder"
+    index:
+        product: firefox
+        job-name: linux64-source-opt
+    treeherder:
+        platform: linux64/opt
+        symbol: tc(Src)
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [build]
+        config:
+            - builds/releng_sub_linux_configs/64_source.py
+        options: []
+        script: "mozharness/scripts/fx_source.py"
+    run-on-projects: []
+    shipping-product: firefox
+
+linux64-fennec-source/opt:
+    description: "Fennec source builder"
+    index:
+        product: fennec
+        job-name: linux64-fennec-source-opt
+    treeherder:
+        platform: android-api-16/opt
+        symbol: tc(Src)
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [build]
+        config:
+            - builds/releng_sub_linux_configs/64_source.py
+        options: []
+        script: "mozharness/scripts/fx_source.py"
+    run-on-projects: []
+    shipping-product: fennec
+
+linux64-devedition-source/opt:
+    description: "Devedition source builder"
+    index:
+        product: devedition
+        job-name: linux64-devedition-source-opt
+    treeherder:
+        platform: linux64-devedition-source/opt
+        symbol: tc(Src)
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [build]
+        config:
+            - builds/releng_sub_linux_configs/64_source.py
+        options: []
+        script: "mozharness/scripts/fx_source.py"
+    run-on-projects: []
+    shipping-product: devedition
--- a/taskcluster/taskgraph/loader/build_signing.py
+++ b/taskcluster/taskgraph/loader/build_signing.py
@@ -7,16 +7,19 @@ from __future__ import absolute_import, 
 from taskgraph.loader.single_dep import loader as base_loader
 
 # XXX: This logic should rely in kind.yml. This hasn't been done in the original
 # patch because it required some heavy changes in single_dep.
 LABELS_WHICH_SHOULD_SIGN_CI_BUILDS = (
     'build-win32/debug', 'build-win32/opt', 'build-win32/pgo',
     'build-win64/debug', 'build-win64/opt', 'build-win64/pgo',
     'build-win32-devedition/opt', 'build-win64-devedition/opt',
+    'release-source-linux64-source/opt',
+    'release-source-linux64-fennec-source/opt',
+    'release-source-linux64-devedition-source/opt',
 )
 
 
 def loader(kind, path, config, params, loaded_tasks):
     jobs = base_loader(kind, path, config, params, loaded_tasks)
 
     for job in jobs:
         dependent_task = job['dependent-task']
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -141,16 +141,22 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
     'linux64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
         "host/bin/mar",
         "host/bin/mbsdiff",
     ],
     'linux-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
         "host/bin/mar",
         "host/bin/mbsdiff",
     ],
+    'linux64-source': [
+    ],
+    'linux64-devedition-source': [
+    ],
+    'linux64-fennec-source': [
+    ],
     'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-aarch64-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-api-16-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-x86-old-id-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-api-16-old-id-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
         "host/bin/mar",
         "host/bin/mbsdiff",
@@ -203,16 +209,28 @@ UPSTREAM_ARTIFACT_SIGNED_PATHS = {
     'linux64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'linux-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
+    'linux64-source': [
+        "SOURCE",
+        "SOURCE.asc",
+    ],
+    'linux64-devedition-source': [
+        "SOURCE",
+        "SOURCE.asc",
+    ],
+    'linux64-fennec-source': [
+        "SOURCE",
+        "SOURCE.asc",
+    ],
     'android-x86-nightly': ["en-US/target.apk"],
     'android-aarch64-nightly': ["en-US/target.apk"],
     'android-api-16-nightly': ["en-US/target.apk"],
     'android-x86-old-id-nightly': ["en-US/target.apk"],
     'android-api-16-old-id-nightly': ["en-US/target.apk"],
     'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.dmg",
         "target.dmg.asc",
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover_source.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover-source task to also append `build` as dependency
+"""
+from __future__ import absolute_import
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def tweak_beetmover_source_dependencies_and_upstream_artifacts(config, jobs):
+    for job in jobs:
+        # HACK1: instead of grabbing SOURCE file from `release-source` task, we
+        # instead take it along with SOURCE.asc directly from the
+        # `release-source-signing`.
+        #
+        # HACK2: This way, we can just overwrite the `build`
+        # dependency, which at this point still is `release-source` task, with
+        # the actual Nightly en-US linux64 build which contains the
+        # `balrog_props` file we're interested in.
+        #
+        # XXX: this hack should go away by either:
+        # * rewriting beetmover transforms to allow more flexibility in deps
+        # * ditch balrog_props in beetmover and rely on in-tree task payload
+
+        if job['attributes']['shipping_product'] == 'firefox':
+            job['dependencies']['build'] = u'build-linux64-nightly/opt'
+        elif job['attributes']['shipping_product'] == 'fennec':
+            job['dependencies']['build'] = u'build-android-api-16-nightly/opt'
+        elif job['attributes']['shipping_product'] == 'devedition':
+            job['dependencies']['build'] = u'build-linux64-devedition-nightly/opt'
+        else:
+            raise NotImplemented(
+                "Unknown shipping_product {} for beetmover_source!".format(
+                    job['attributes']['shipping_product']
+                )
+            )
+        upstream_artifacts = job['worker']['upstream-artifacts']
+        for artifact in upstream_artifacts:
+            if artifact['taskType'] == 'build':
+                artifact['paths'].append(u'public/build/balrog_props.json')
+                break
+
+        yield job
--- a/taskcluster/taskgraph/transforms/build_signing.py
+++ b/taskcluster/taskgraph/transforms/build_signing.py
@@ -39,17 +39,18 @@ def add_signed_routes(config, jobs):
 def define_upstream_artifacts(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
         build_platform = dep_job.attributes.get('build_platform')
 
         artifacts_specifications = generate_specifications_of_artifacts_to_sign(
             build_platform,
             dep_job.attributes.get('nightly'),
-            keep_locale_template=False
+            keep_locale_template=False,
+            kind=config.kind,
         )
 
         if 'android' in build_platform:
             # We're in the job that creates both multilocale and en-US APKs
             artifacts_specifications[0]['artifacts'].append('public/build/en-US/target.apk')
 
         job['upstream-artifacts'] = [{
             'taskId': {'task-reference': '<build>'},
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -26,16 +26,17 @@ JOB_NAME_WHITELIST = set([
     'android-findbugs',
     'android-lint',
     'android-test',
     'android-x86-old-id-opt',
     'android-x86-opt',
     'browser-haz-debug',
     'linux-debug',
     'linux-devedition',
+    'linux-devedition-opt',
     'linux-devedition-nightly-repackage',
     'linux-devedition-nightly-repackage-signing',
     'linux-nightly-repackage',
     'linux-nightly-repackage-signing',
     'linux-opt',
     'linux-pgo',
     'linux-rusttests-opt',
     'linux-rusttests-debug',
@@ -47,34 +48,39 @@ JOB_NAME_WHITELIST = set([
     'linux64-base-toolchains-debug',
     'linux64-base-toolchains-opt',
     'linux64-fuzzing-asan-opt',
     'linux64-fuzzing-debug',
     'linux64-ccov-opt',
     'linux64-clang-tidy',
     'linux64-debug',
     'linux64-devedition',
+    'linux64-devedition-opt',
     'linux64-devedition-nightly-repackage',
     'linux64-devedition-nightly-repackage-signing',
     'linux64-jsdcov-opt',
     'linux64-nightly-repackage',
     'linux64-nightly-repackage-signing',
     'linux64-noopt-debug',
     'linux64-opt',
     'linux64-pgo',
     'linux64-rusttests-opt',
     'linux64-rusttests-debug',
     'linux64-searchfox-debug',
     'linux64-st-an-debug',
     'linux64-st-an-opt',
     'linux64-valgrind-opt',
     'linux64-dmd-opt',
+    'linux64-source-opt',
+    'linux64-devedition-source-opt',
+    'linux64-fennec-source-opt',
     'macosx64-add-on-devel',
     'macosx64-clang-tidy',
     'macosx64-debug',
+    'macosx64-devedition-opt',
     'macosx64-nightly-repackage',
     'macosx64-nightly-repackage-signing',
     'macosx64-noopt-debug',
     'macosx64-opt',
     'macosx64-devedition-nightly-repackage',
     'macosx64-devedition-nightly-repackage-signing',
     'macosx64-st-an-debug',
     'macosx64-st-an-opt',
--- a/taskcluster/taskgraph/util/signed_artifacts.py
+++ b/taskcluster/taskgraph/util/signed_artifacts.py
@@ -4,19 +4,26 @@
 """
 Defines artifacts to sign before repackage.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 
 def generate_specifications_of_artifacts_to_sign(
-    build_platform, is_nightly=False, keep_locale_template=True
+    build_platform, is_nightly=False, keep_locale_template=True, kind=None
 ):
-    if 'android' in build_platform:
+    if kind == 'release-source-signing':
+        artifacts_specifications = [{
+            'artifacts': [
+                'public/build/SOURCE'
+            ],
+            'formats': ['gpg'],
+        }]
+    elif 'android' in build_platform:
         artifacts_specifications = [{
             'artifacts': [
                 'public/build/{locale}/target.apk',
             ],
             'formats': ['jar'],
         }]
     # XXX: Mars aren't signed here (on any platform) because internals will be
     # signed at after this stage of the release
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_source.py
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_source.py
@@ -1,19 +1,21 @@
+import os
+
 config = {
-    'default_actions': [
-        'clobber',
-        'clone-tools',
-        'checkout-sources',
-        'setup-mock',
-        'package-source',
-        'generate-source-signing-manifest',
-    ],
-    'stage_platform': 'source',  # Not used, but required by the script
-    'buildbot_json_path': 'buildprops.json',
-    'app_ini_path': 'FAKE',  # Not used, but required by the script
-    'env': {
-        'MOZ_OBJDIR': '%(abs_obj_dir)s',
-        'TINDERBOX_OUTPUT': '1',
-        'LC_ALL': 'C',
-    },
-    'mozconfig_variant': 'source',
+    "tmpl": """The binaries in this directory are made available to you under the Mozilla Public License v.2
+(MPL 2):
+http://www.mozilla.org/MPL/2.0/
+
+The source code used to build these binaries is specified by the following unique URL:
+{repo}/rev/{revision}
+
+    zip: {repo}/archive/{revision}.zip
+    tar.gz: {repo}/archive/{revision}.tar.gz
+    tar.bz2: {repo}/archive/{revision}.tar.bz2
+
+Instructions for downloading and building this source code can be found here:
+https://developer.mozilla.org/en/Mozilla_Source_Code_%28Mercurial%29
+""",
+    "out_path": "artifacts/SOURCE",
+    "repo": os.environ["GECKO_HEAD_REPOSITORY"],
+    "revision": os.environ["GECKO_HEAD_REV"],
 }
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/fx_source.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""fx_source.py.
+
+Create a source readme.
+
+"""
+import sys
+import os
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.script import BaseScript
+from mozharness.base.log import FATAL
+
+
+class FxSource(BaseScript):
+    config_options = [
+        [["--repo"], {
+            "dest": "repo",
+            "help": "the URL to the source repo",
+        }],
+        [["--revision"], {
+            "dest": "revision",
+            "help": "the revision of the source",
+        }],
+        [["--disable-mock"], {
+            "dest": "disable_mock",
+            "action": "store_true",
+            "help": "dummy option",
+        }],
+        [["--scm-level"], {
+            "dest": "scm_level",
+            "help": "dummy option",
+        }],
+        [["--branch"], {
+            "dest": "branch",
+            "help": "dummy option",
+        }],
+        [["--build-pool"], {
+            "dest": "build_pool",
+            "help": "dummy option",
+        }],
+    ]
+
+    def __init__(self):
+        buildscript_kwargs = {
+            'config_options': self.config_options,
+            'all_actions': [
+                'build',
+            ],
+            'require_config_file': True,
+        }
+        super(FxSource, self).__init__(**buildscript_kwargs)
+
+    def build(self):
+        c = self.config
+        contents = c['tmpl'].format(**c)
+        self.write_to_file(c['out_path'], contents, create_parent_dir=True,
+                           error_level=FATAL)
+
+
+if __name__ == '__main__':
+    fx_source = FxSource()
+    fx_source.run_and_exit()