Bug 1458385: Repackage windows builds on linux. r=glandium,Callek
☠☠ backed out by 4e2ea1a75e87 ☠ ☠
authorTom Prince <mozilla@hocat.ca>
Wed, 27 Mar 2019 22:30:54 +0000
changeset 466469 8036c03c3b5139a3b79e2e21c58abee6412f39cd
parent 466468 99fa5539c2bda621abac134300428129969b2cdf
child 466470 173bc2937a7beb6e5aa516a40000e0ee2ea1ff0d
push id35768
push useropoprus@mozilla.com
push dateThu, 28 Mar 2019 09:55:54 +0000
treeherdermozilla-central@c045dd97faf2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium, Callek
bugs1458385
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1458385: Repackage windows builds on linux. r=glandium,Callek This only uses cross-platform tools, so switch to running these on linux, which cuts the runtime down from ~20m to ~3m. Differential Revision: https://phabricator.services.mozilla.com/D1080
taskcluster/ci/repackage-l10n/kind.yml
taskcluster/ci/repackage-msi/kind.yml
taskcluster/ci/repackage/kind.yml
taskcluster/taskgraph/transforms/repackage.py
taskcluster/taskgraph/transforms/repackage_partner.py
testing/mozharness/configs/repackage/win32_partner.py
testing/mozharness/configs/repackage/win32_signed.py
testing/mozharness/configs/repackage/win64_partner.py
testing/mozharness/configs/repackage/win64_signed.py
testing/mozharness/scripts/repackage.py
--- a/taskcluster/ci/repackage-l10n/kind.yml
+++ b/taskcluster/ci/repackage-l10n/kind.yml
@@ -32,16 +32,19 @@ only-for-build-platforms:
     - linux-devedition-nightly/opt
     - linux64-devedition-nightly/opt
     - macosx64-devedition-nightly/opt
     - win32-devedition-nightly/opt
     - win64-devedition-nightly/opt
     - win64-aarch64-devedition-nightly/opt
 
 job-template:
+    worker-type: 'aws-provisioner-v1/gecko-{level}-b-linux'
+    worker:
+        docker-image: {"in-tree": "debian7-amd64-build"}
     mozharness:
         config:
             by-build-platform:
                 linux-.*:
                     - repackage/base.py
                     - repackage/linux32_signed.py
                 linux64-.*:
                     - repackage/base.py
@@ -72,8 +75,12 @@ job-template:
                     win64\b.*: [mar, mar-bz2, installer]
             default:
                 by-build-platform:
                     linux.*: [mar]
                     linux4\b.*: [mar]
                     macosx64\b.*: [mar, dmg]
                     win32\b.*: [mar, installer]
                     win64\b.*: [mar, installer]
+    fetches:
+        toolchain:
+            - linux64-mar-tools
+            - linux64-upx
--- a/taskcluster/ci/repackage-msi/kind.yml
+++ b/taskcluster/ci/repackage-msi/kind.yml
@@ -20,17 +20,20 @@ only-for-build-platforms:
     - win32-shippable/opt
     - win32-nightly/opt
     - win64-shippable/opt
     - win64-nightly/opt
     - win32-devedition-nightly/opt
     - win64-devedition-nightly/opt
 
 job-template:
+    worker-type: 'aws-provisioner-v1/gecko-{level}-b-win2012'
     mozharness:
+        use-magic-mh-args: false
+        use-caches: false
         config:
             by-build-platform:
                 win32\b.*:
                     - repackage/base.py
                     - repackage/win32_sfx_stub.py
                     - repackage/win32_signed.py
                 win64(?!-aarch64)\b.*:
                     - repackage/base.py
--- a/taskcluster/ci/repackage/kind.yml
+++ b/taskcluster/ci/repackage/kind.yml
@@ -42,16 +42,19 @@ only-for-build-platforms:
     - win64-aarch64-shippable/opt
     - win64-aarch64-nightly/opt
     - win64-aarch64-devedition-nightly/opt
     - win64/opt
     - win64-pgo/opt
     - win64-asan-reporter-nightly/opt
 
 job-template:
+    worker-type: 'aws-provisioner-v1/gecko-{level}-b-linux'
+    worker:
+        docker-image: {"in-tree": "debian7-amd64-build"}
     mozharness:
         config:
             by-build-platform:
                 linux\b.*:
                     - repackage/base.py
                     - repackage/linux32_signed.py
                 linux64\b.*:
                     - repackage/base.py
@@ -82,8 +85,12 @@ job-template:
                     win64\b.*: [mar, mar-bz2, installer]
             default:
                 by-build-platform:
                     linux.*: [mar]
                     linux4\b.*: [mar]
                     macosx64\b.*: [mar, dmg]
                     win32\b.*: [mar, installer]
                     win64\b.*: [mar, installer]
+    fetches:
+        toolchain:
+            - linux64-mar-tools
+            - linux64-upx
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -15,26 +15,29 @@ from taskgraph.util.attributes import co
 from taskgraph.util.schema import (
     optionally_keyed_by,
     resolve_keyed_by,
 )
 from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.util.platforms import archive_format, executable_extension, architecture
 from taskgraph.util.workertypes import worker_type_implementation
 from taskgraph.transforms.job import job_description_schema
-from voluptuous import Required, Optional
+from voluptuous import Required, Optional, Extra
 
 
 packaging_description_schema = schema.extend({
     # depname is used in taskref's to identify the taskID of the signed things
     Required('depname', default='build'): basestring,
 
     # unique label to describe this repackaging task
     Optional('label'): basestring,
 
+    Optional('worker-type'): basestring,
+    Optional('worker'): object,
+
     # treeherder is allowed here to override any defaults we use for repackaging.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
     # below transforms for defaults of various values.
     Optional('treeherder'): job_description_schema['treeherder'],
 
     # If a l10n task, the corresponding locale
     Optional('locale'): basestring,
 
@@ -51,16 +54,17 @@ packaging_description_schema = schema.ex
     Optional('shipping-product'): job_description_schema['shipping-product'],
     Optional('shipping-phase'): job_description_schema['shipping-phase'],
 
     Required('package-formats'): optionally_keyed_by(
         'build-platform', 'release-type', [basestring]),
 
     # All l10n jobs use mozharness
     Required('mozharness'): {
+        Extra: object,
         # Config files passed to the mozharness script
         Required('config'): optionally_keyed_by('build-platform', [basestring]),
 
         # Additional paths to look for mozharness configs in. These should be
         # relative to the base of the source checkout
         Optional('config-paths'): [basestring],
 
         # if true, perform a checkout of a comm-central based branch inside the
@@ -200,20 +204,16 @@ def make_repackage_description(config, j
 def make_job_description(config, jobs):
     for job in jobs:
         dep_job = job['primary-dependency']
         dependencies = {dep_job.attributes.get('kind'): dep_job.label}
         if len(dep_job.dependencies) > 1 and not config.kind == 'repackage-msi':
             # repackage-signing can end up with multiple deps...
             raise NotImplementedError(
                 "Can't repackage a signing task with multiple dependencies")
-        signing_dependencies = dep_job.dependencies
-        # This is so we get the build task in our dependencies to
-        # have better beetmover support.
-        dependencies.update(signing_dependencies)
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes['repackage_type'] = 'repackage'
 
         locale = attributes.get('locale', job.get('locale'))
         if locale:
             attributes['locale'] = locale
 
@@ -225,39 +225,29 @@ def make_job_description(config, jobs):
         dep_th_platform = dep_job.task.get('extra', {}).get('treeherder-platform')
         treeherder.setdefault('platform', dep_th_platform)
         treeherder.setdefault('tier', 1)
         treeherder.setdefault('kind', 'build')
 
         if config.kind == 'repackage-msi':
             treeherder['symbol'] = 'MSI({})'.format(locale or 'N')
 
-        build_task = None
         signing_task = None
         repackage_signing_task = None
         for dependency in dependencies.keys():
             if 'repackage-signing' in dependency:
                 repackage_signing_task = dependency
             elif 'signing' in dependency:
                 signing_task = dependency
-            else:
-                build_task = dependency
 
         _fetch_subst_locale = 'en-US'
         if locale:
-            # XXXCallek: todo: rewrite dependency finding
-            # Use string splice to strip out 'nightly-l10n-' .. '-<chunk>/opt'
-            # We need this additional dependency to support finding the mar binary
-            # Which is needed in order to generate a new complete.mar
-            dependencies['build'] = "build-{}/opt".format(
-                dependencies[build_task][13:dependencies[build_task].rfind('-')])
-            build_task = 'build'
             _fetch_subst_locale = locale
 
-        level = config.params['level']
+        worker_type = job['worker-type']
         build_platform = attributes['build_platform']
 
         use_stub = attributes.get('stub-installer')
 
         repackage_config = []
         package_formats = job.get('package-formats')
         if use_stub and not repackage_signing_task:
             # if repackage_signing_task doesn't exists, generate the stub installer
@@ -293,42 +283,28 @@ def make_job_description(config, jobs):
             'job-script': 'taskcluster/scripts/builder/repackage.sh',
             'actions': ['setup', 'repackage'],
             'extra-workspace-cache-key': 'repackage',
             'extra-config': {
                 'repackage_config': repackage_config,
             },
         })
 
-        worker = {
+        worker = job.get('worker', {})
+        worker.update({
             'chain-of-trust': True,
             'max-run-time': 7200 if build_platform.startswith('win') else 3600,
             # Don't add generic artifact directory.
             'skip-artifacts': True,
-        }
+        })
 
         if locale:
             # Make sure we specify the locale-specific upload dir
             worker.setdefault('env', {}).update(LOCALE=locale)
 
-        if build_platform.startswith('win'):
-            worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
-            run['use-magic-mh-args'] = False
-            run['use-caches'] = False
-        else:
-            if build_platform.startswith(('linux', 'macosx')):
-                worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
-            else:
-                raise NotImplementedError(
-                    'Unsupported build_platform: "{}"'.format(build_platform)
-                )
-
-            run['tooltool-downloads'] = 'internal'
-            worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
-
         worker['artifacts'] = _generate_task_output_files(
             dep_job, worker_type_implementation(config.graph_config, worker_type),
             repackage_config=repackage_config,
             locale=locale,
         )
 
         description = (
             "Repackaging for locale '{locale}' for build '"
@@ -346,34 +322,34 @@ def make_job_description(config, jobs):
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
             'routes': job.get('routes', []),
             'extra': job.get('extra', {}),
             'worker': worker,
             'run': run,
-            'fetches': _generate_download_config(dep_job, build_platform, build_task,
+            'fetches': _generate_download_config(dep_job, build_platform,
                                                  signing_task, repackage_signing_task,
                                                  locale=locale,
                                                  project=config.params["project"],
                                                  existing_fetch=job.get('fetches')),
             'release-artifacts': [artifact['name'] for artifact in worker['artifacts']]
         }
 
         if build_platform.startswith('macosx'):
             task['toolchains'] = [
                 'linux64-libdmg',
                 'linux64-hfsplus',
                 'linux64-node',
             ]
         yield task
 
 
-def _generate_download_config(task, build_platform, build_task, signing_task,
+def _generate_download_config(task, build_platform, signing_task,
                               repackage_signing_task, locale=None, project=None,
                               existing_fetch=None):
     locale_path = '{}/'.format(locale) if locale else ''
     fetch = {}
     if existing_fetch:
         fetch.update(existing_fetch)
 
     if repackage_signing_task and build_platform.startswith('win'):
@@ -385,30 +361,26 @@ def _generate_download_config(task, buil
     elif build_platform.startswith('linux') or build_platform.startswith('macosx'):
         fetch.update({
             signing_task: [
                 {
                     'artifact': '{}target{}'.format(locale_path, archive_format(build_platform)),
                     'extract': False,
                 },
             ],
-            "toolchain": ["linux64-mar-tools"],
         })
     elif build_platform.startswith('win'):
         fetch.update({
             signing_task: [
                 {
                     'artifact': '{}target.zip'.format(locale_path),
                     'extract': False,
                 },
                 '{}setup.exe'.format(locale_path),
             ],
-            build_task: [
-                'host/bin/mar.exe',
-            ],
         })
 
         use_stub = task.attributes.get('stub-installer')
         if use_stub:
             fetch[signing_task].append('{}setup-stub.exe'.format(locale_path))
 
     if fetch:
         return fetch
--- a/taskcluster/taskgraph/transforms/repackage_partner.py
+++ b/taskcluster/taskgraph/transforms/repackage_partner.py
@@ -162,29 +162,18 @@ def make_job_description(config, jobs):
             'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False,
             'env': {
                 'REPACK_ID': repack_id,
             },
             # Don't add generic artifact directory.
             'skip-artifacts': True,
         }
 
-        if build_platform.startswith('win'):
-            worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
-            run['use-magic-mh-args'] = False
-        else:
-            if build_platform.startswith('macosx'):
-                worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
-            else:
-                raise NotImplementedError(
-                    'Unsupported build_platform: "{}"'.format(build_platform)
-                )
-
-            run['tooltool-downloads'] = 'internal'
-            worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
+        worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
+        worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
 
         worker['artifacts'] = _generate_task_output_files(
             dep_job, worker_type_implementation(config.graph_config, worker_type),
             repackage_config, partner=repack_id,
         )
 
         description = (
             "Repackaging for repack_id '{repack_id}' for build '"
--- a/testing/mozharness/configs/repackage/win32_partner.py
+++ b/testing/mozharness/configs/repackage/win32_partner.py
@@ -1,14 +1,14 @@
 import os
 
 platform = "win32"
 
 config = {
     "repack_id": os.environ.get("REPACK_ID"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
     'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+    'run_configure': False,
 
-    'run_configure': False,
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }
--- a/testing/mozharness/configs/repackage/win32_signed.py
+++ b/testing/mozharness/configs/repackage/win32_signed.py
@@ -1,14 +1,13 @@
 import os
 
 platform = "win32"
 
 config = {
     "locale": os.environ.get("LOCALE"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
-    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+    'run_configure': False,
 
-    'run_configure': False,
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }
--- a/testing/mozharness/configs/repackage/win64_partner.py
+++ b/testing/mozharness/configs/repackage/win64_partner.py
@@ -1,14 +1,14 @@
 import os
 
 platform = "win64"
 
 config = {
     "repack_id": os.environ.get("REPACK_ID"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
     'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+    'run_configure': False,
 
-    'run_configure': False,
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }
--- a/testing/mozharness/configs/repackage/win64_signed.py
+++ b/testing/mozharness/configs/repackage/win64_signed.py
@@ -1,14 +1,13 @@
 import os
 
 platform = "win64"
 
 config = {
     "locale": os.environ.get("LOCALE"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
-    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+    'run_configure': False,
 
-    'run_configure': False,
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }
--- a/testing/mozharness/scripts/repackage.py
+++ b/testing/mozharness/scripts/repackage.py
@@ -88,16 +88,17 @@ class Repackage(BaseScript):
                 ])
             command.extend([
                 '--output', os.path.join(dirs['abs_output_dir'], repack_config['output']),
             ])
             self.run_command(
                 command=command,
                 cwd=dirs['abs_mozilla_dir'],
                 halt_on_failure=True,
+                env=self.query_env(),
             )
 
     def _run_tooltool(self):
         config = self.config
         dirs = self.query_abs_dirs()
         toolchains = os.environ.get('MOZ_TOOLCHAINS')
         manifest_src = os.environ.get('TOOLTOOL_MANIFEST')
         if not manifest_src: