Bug 1458395 - Do not use stub installer on esr60. r=aki a=release CLOSED TREE
authorJustin Wood <Callek@gmail.com>
Tue, 01 May 2018 19:32:38 -0400
changeset 790949 a51793b8f67b1c4fab8572947fb829b0f1e77f2a
parent 790948 3a617bd3456bf23438b25a8d0b13c9976858226e
child 790950 e71bb842a52f3e7170fa19417fdaf8b18b169e3e
push id108656
push userbmo:jlorenzo@mozilla.com
push dateThu, 03 May 2018 08:03:49 +0000
reviewersaki, release
bugs1458395
milestone60.0
Bug 1458395 - Do not use stub installer on esr60. r=aki a=release CLOSED TREE On ESR[60] stub installer isn't designed to work, and we expect most users of esr will have no use for a stub installer. However this poses some problems where the taskgraph assumes that any Nightly generates a stub installer and thus it should be available along the way. The patch hardcodes the list of branches that do not need a stub installer, though we'll want to clean up this specification in some way, as Thunderbird will likely need it to be cleaner and we may want on-change builds to use this logic (e.g. for on-change l10n). Differential Revision: https://phabricator.services.mozilla.com/D1082
taskcluster/ci/release-bouncer-sub/kind.yml
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/transforms/bouncer_submission.py
taskcluster/taskgraph/transforms/build_signing.py
taskcluster/taskgraph/transforms/nightly_l10n_signing.py
taskcluster/taskgraph/transforms/repackage.py
taskcluster/taskgraph/transforms/repackage_signing.py
taskcluster/taskgraph/util/signed_artifacts.py
testing/mozharness/configs/repackage/win32_signed.py
--- a/taskcluster/ci/release-bouncer-sub/kind.yml
+++ b/taskcluster/ci/release-bouncer-sub/kind.yml
@@ -42,15 +42,20 @@ jobs:
    fennec:
       bouncer-platforms: ['android', 'android-x86']
       bouncer-products: ['apk']
       shipping-product: fennec
       locales-file: mobile/locales/l10n-changesets.json
 
    firefox:
       bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
-      bouncer-products: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer']
+      bouncer-products:
+         by-project:
+            default: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer']
+            # No stub installer in esr60
+            mozilla-esr60: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar']
+            jamun: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar']
       shipping-product: firefox
 
    firefox-rc:
       bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
       bouncer-products: ['partial-mar-candidates']
       shipping-product: firefox
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -256,17 +256,17 @@ def make_task_description(config, jobs):
             'shipping-product': job.get('shipping-product'),
         }
 
         yield task
 
 
 def generate_upstream_artifacts(job, build_task_ref, build_signing_task_ref,
                                 repackage_task_ref, repackage_signing_task_ref,
-                                platform, locale=None):
+                                platform, locale=None, project=None):
 
     build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
     build_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
     repackage_mapping = UPSTREAM_ARTIFACT_REPACKAGE_PATHS
     repackage_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS
 
     artifact_prefix = get_artifact_prefix(job)
     if locale:
@@ -292,20 +292,29 @@ def generate_upstream_artifacts(job, bui
     for ref, tasktype, mapping in zip(task_refs, tasktypes, mapping):
         plarform_was_previously_matched_by_regex = None
         for platform_regex, paths in mapping.iteritems():
             if platform_regex.match(platform) is not None:
                 _check_platform_matched_only_one_regex(
                     tasktype, platform, plarform_was_previously_matched_by_regex, platform_regex
                 )
                 if paths:
+                    usable_paths = paths[:]
+
+                    no_stub = ("mozilla-esr60", "jamun")
+                    if project in no_stub:
+                        # Stub installer is only generated on win32 and not on esr
+                        # XXX We really should have a better solution for this
+                        if 'target.stub-installer.exe' in usable_paths:
+                            usable_paths.remove('target.stub-installer.exe')
                     upstream_artifacts.append({
                         "taskId": {"task-reference": ref},
                         "taskType": tasktype,
-                        "paths": ["{}/{}".format(artifact_prefix, path) for path in paths],
+                        "paths": ["{}/{}".format(artifact_prefix, path)
+                                  for path in usable_paths],
                         "locale": locale or "en-US",
                     })
                 plarform_was_previously_matched_by_regex = platform_regex
 
     return upstream_artifacts
 
 
 def generate_partials_upstream_artifacts(job, artifacts, platform, locale=None):
@@ -381,17 +390,18 @@ def make_task_worker(config, jobs):
         repackage_task_ref = "<" + str(repackage_task) + ">"
         repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">"
 
         worker = {
             'implementation': 'beetmover',
             'release-properties': craft_release_properties(config, job),
             'upstream-artifacts': generate_upstream_artifacts(
                 job, build_task_ref, build_signing_task_ref, repackage_task_ref,
-                repackage_signing_task_ref, platform, locale
+                repackage_signing_task_ref, platform, locale,
+                project=config.params['project']
             ),
         }
         if locale:
             worker["locale"] = locale
         job["worker"] = worker
 
         yield job
 
--- a/taskcluster/taskgraph/transforms/bouncer_submission.py
+++ b/taskcluster/taskgraph/transforms/bouncer_submission.py
@@ -87,16 +87,19 @@ transforms = TransformSequence()
 def make_task_worker(config, jobs):
     for job in jobs:
         resolve_keyed_by(
             job, 'worker-type', item_name=job['name'], project=config.params['project']
         )
         resolve_keyed_by(
             job, 'scopes', item_name=job['name'], project=config.params['project']
         )
+        resolve_keyed_by(
+            job, 'bouncer-products', item_name=job['name'], project=config.params['project']
+        )
 
         # No need to filter out ja-JP-mac, we need to upload both; but we do
         # need to filter out the platforms they come with
         all_locales = sorted([
             locale
             for locale in parse_locales_file(job['locales-file']).keys()
             if locale not in ('linux', 'win32', 'osx')
         ])
--- a/taskcluster/taskgraph/transforms/build_signing.py
+++ b/taskcluster/taskgraph/transforms/build_signing.py
@@ -41,16 +41,17 @@ def define_upstream_artifacts(config, jo
     for job in jobs:
         dep_job = job['dependent-task']
         build_platform = dep_job.attributes.get('build_platform')
 
         artifacts_specifications = generate_specifications_of_artifacts_to_sign(
             dep_job,
             keep_locale_template=False,
             kind=config.kind,
+            project=config.params["project"],
         )
 
         if 'android' in build_platform:
             # We're in the job that creates both multilocale and en-US APKs
             artifacts_specifications[0]['artifacts'].append(
                 get_artifact_path(dep_job, 'en-US/target.apk')
             )
 
--- a/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
+++ b/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
@@ -34,17 +34,18 @@ def make_signing_description(config, job
 
 @transforms.add
 def define_upstream_artifacts(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
 
         locale_specifications = generate_specifications_of_artifacts_to_sign(
             dep_job,
-            keep_locale_template=True
+            keep_locale_template=True,
+            project=config.params["project"],
         )
 
         upstream_artifacts = []
         for spec in locale_specifications:
             upstream_artifacts.append({
                 'taskId': {'task-reference': '<unsigned-repack>'},
                 'taskType': 'l10n',
                 # Set paths based on artifacts in the specs (above) one per
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -180,18 +180,21 @@ def make_job_description(config, jobs):
             'script': 'mozharness/scripts/repackage.py',
             'job-script': 'taskcluster/scripts/builder/repackage.sh',
             'actions': ['download_input', 'setup', 'repackage'],
             'extra-workspace-cache-key': 'repackage',
         })
 
         worker = {
             'env': _generate_task_env(dep_job, build_platform, build_task_ref,
-                                      signing_task_ref, locale=locale),
-            'artifacts': _generate_task_output_files(dep_job, build_platform, locale=locale),
+                                      signing_task_ref, locale=locale,
+                                      project=config.params["project"]),
+            'artifacts': _generate_task_output_files(dep_job, build_platform,
+                                                     locale=locale,
+                                                     project=config.params["project"]),
             'chain-of-trust': True,
             'max-run-time': 7200 if build_platform.startswith('win') else 3600,
         }
 
         if locale:
             # Make sure we specify the locale-specific upload dir
             worker['env'].update(LOCALE=locale)
 
@@ -237,17 +240,18 @@ def make_job_description(config, jobs):
         if build_platform.startswith('macosx'):
             task['toolchains'] = [
                 'linux64-libdmg',
                 'linux64-hfsplus',
             ]
         yield task
 
 
-def _generate_task_env(task, build_platform, build_task_ref, signing_task_ref, locale=None):
+def _generate_task_env(task, build_platform, build_task_ref, signing_task_ref, locale=None,
+                       project=None):
     mar_prefix = get_taskcluster_artifact_prefix(
         task, build_task_ref, postfix='host/bin/', locale=None
     )
     signed_prefix = get_taskcluster_artifact_prefix(task, signing_task_ref, locale=locale)
 
     if build_platform.startswith('linux') or build_platform.startswith('macosx'):
         tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz'
         return {
@@ -258,27 +262,32 @@ def _generate_task_env(task, build_platf
         }
     elif build_platform.startswith('win'):
         task_env = {
             'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)},
             'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)},
             'UNSIGNED_MAR': {'task-reference': '{}mar.exe'.format(mar_prefix)},
         }
 
-        # Stub installer is only generated on win32
-        if '32' in build_platform:
-            task_env['SIGNED_SETUP_STUB'] = {
-                'task-reference': '{}setup-stub.exe'.format(signed_prefix),
-            }
+        no_stub = ("mozilla-esr60", "jamun")
+        if project in no_stub:
+            # Stub installer is only generated on win32 and not on esr
+            task_env['NO_STUB_INSTALLER'] = '1'
+        else:
+            # Stub installer is only generated on win32
+            if '32' in build_platform:
+                task_env['SIGNED_SETUP_STUB'] = {
+                    'task-reference': '{}setup-stub.exe'.format(signed_prefix),
+                }
         return task_env
 
     raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
 
 
-def _generate_task_output_files(task, build_platform, locale=None):
+def _generate_task_output_files(task, build_platform, locale=None, project=None):
     locale_output_path = '{}/'.format(locale) if locale else ''
     artifact_prefix = get_artifact_prefix(task)
 
     if build_platform.startswith('linux') or build_platform.startswith('macosx'):
         output_files = [{
             'type': 'file',
             'path': '/builds/worker/workspace/build/artifacts/{}target.complete.mar'
                     .format(locale_output_path),
@@ -299,18 +308,19 @@ def _generate_task_output_files(task, bu
             'path': '{}/{}target.installer.exe'.format(artifact_prefix, locale_output_path),
             'name': '{}/{}target.installer.exe'.format(artifact_prefix, locale_output_path),
         }, {
             'type': 'file',
             'path': '{}/{}target.complete.mar'.format(artifact_prefix, locale_output_path),
             'name': '{}/{}target.complete.mar'.format(artifact_prefix, locale_output_path),
         }]
 
-        # Stub installer is only generated on win32
-        if '32' in build_platform:
+        # Stub installer is only generated on win32 and not on esr
+        no_stub = ("mozilla-esr60", "jamun")
+        if 'win32' in build_platform and project not in no_stub:
             output_files.append({
                 'type': 'file',
                 'path': '{}/{}target.stub-installer.exe'.format(
                     artifact_prefix, locale_output_path
                 ),
                 'name': '{}/{}target.stub-installer.exe'.format(
                     artifact_prefix, locale_output_path
                 ),
--- a/taskcluster/taskgraph/transforms/repackage_signing.py
+++ b/taskcluster/taskgraph/transforms/repackage_signing.py
@@ -107,18 +107,20 @@ def make_repackage_signing_description(c
                 "taskType": "repackage",
                 "paths": [
                     get_artifact_path(dep_job, "{}target.installer.exe".format(locale_str)),
                 ],
                 "formats": ["sha2signcode"]
             })
             scopes.append(add_scope_prefix(config, "signing:format:sha2signcode"))
 
-            # Stub installer is only generated on win32
-            if '32' in build_platform:
+            # Stub installer is only generated on win32 and not on esr
+            no_stub = ("mozilla-esr60", "jamun")
+            if 'win32' in build_platform and not config.params["project"] in no_stub:
+                # TODO: fix the project hint to be a better design
                 upstream_artifacts.append({
                     "taskId": {"task-reference": "<repackage>"},
                     "taskType": "repackage",
                     "paths": [
                         get_artifact_path(
                             dep_job, "{}target.stub-installer.exe".format(locale_str)
                         ),
                     ],
--- a/taskcluster/taskgraph/util/signed_artifacts.py
+++ b/taskcluster/taskgraph/util/signed_artifacts.py
@@ -10,17 +10,17 @@ from taskgraph.util.taskcluster import g
 
 
 def is_partner_kind(kind):
     if kind and kind.startswith(('release-partner', 'release-eme-free')):
         return True
 
 
 def generate_specifications_of_artifacts_to_sign(
-    task, keep_locale_template=True, kind=None
+    task, keep_locale_template=True, kind=None, project=None
 ):
     build_platform = task.attributes.get('build_platform')
     is_nightly = task.attributes.get('nightly')
     if kind == 'release-source-signing':
         artifacts_specifications = [{
             'artifacts': [
                 get_artifact_path(task, 'source.tar.xz')
             ],
@@ -51,17 +51,20 @@ def generate_specifications_of_artifacts
             ],
             'formats': ['sha2signcode'],
         }, {
             'artifacts': [
                 get_artifact_path(task, '{locale}/target.zip'),
             ],
             'formats': ['sha2signcode', 'widevine'],
         }]
-        if 'win32' in build_platform and is_nightly:
+        no_stub = ("mozilla-esr60", "jamun")
+        if 'win32' in build_platform and is_nightly and project not in no_stub:
+            # TODO: fix the project hint to be a better design
+            # We don't build stub installer on esr, so we don't want to sign it
             artifacts_specifications[0]['artifacts'] += [
                 get_artifact_path(task, '{locale}/setup-stub.exe')
             ]
     elif 'linux' in build_platform:
         artifacts_specifications = [{
             'artifacts': [get_artifact_path(task, '{locale}/target.tar.bz2')],
             'formats': ['gpg', 'widevine'],
         }]
--- a/testing/mozharness/configs/repackage/win32_signed.py
+++ b/testing/mozharness/configs/repackage/win32_signed.py
@@ -1,40 +1,49 @@
 import os
 
 platform = "win32"
 
 download_config = {
         "target.zip": os.environ.get("SIGNED_ZIP"),
         "setup.exe": os.environ.get("SIGNED_SETUP"),
         "mar.exe": os.environ.get("UNSIGNED_MAR"),
+}
+
+if not os.environ.get("NO_STUB_INSTALLER"):
+    # Some channels, like esr don't build a stub installer
+    download_config.update({
         # stub installer is only built on win32
         "setup-stub.exe": os.environ.get("SIGNED_SETUP_STUB"),
-    }
+    })
 
 repackage_config = [[
         "installer",
         "--package-name", "firefox",
         "--package", "{abs_work_dir}\\inputs\\target.zip",
         "--tag", "{abs_mozilla_dir}\\browser\\installer\\windows\\app.tag",
         "--setupexe", "{abs_work_dir}\\inputs\\setup.exe",
         "-o", "{output_home}\\target.installer.exe",
         "--sfx-stub", "other-licenses/7zstub/firefox/7zSD.sfx",
     ], [
         "mar",
         "-i", "{abs_work_dir}\\inputs\\target.zip",
         "--mar", "{abs_work_dir}\\inputs\\mar.exe",
         "-o", "{output_home}\\target.complete.mar",
-    ], [
+    ]]
+
+if not os.environ.get("NO_STUB_INSTALLER"):
+    # Some channels, like esr don't build a stub installer
+    repackage_config.append([
         "installer",
         "--tag", "{abs_mozilla_dir}\\browser\\installer\\windows\\stub.tag",
-         "--setupexe", "{abs_work_dir}\\inputs\\setup-stub.exe",
-         "-o", "{output_home}\\target.stub-installer.exe",
+        "--setupexe", "{abs_work_dir}\\inputs\\setup-stub.exe",
+        "-o", "{output_home}\\target.stub-installer.exe",
         "--sfx-stub", "other-licenses/7zstub/firefox/7zSD.sfx",
-    ]]
+    ])
 
 config = {
     "input_home": "{abs_work_dir}\\inputs",
     "output_home": "{base_work_dir}\\public\\build{locale}",
 
     "locale": os.environ.get("LOCALE"),
 
     "download_config": download_config,