Bug 1458385: Repackage windows builds on linux. r=glandium,Callek
authorTom Prince <mozilla@hocat.ca>
Sat, 06 Apr 2019 05:49:55 +0000
changeset 468595 09338587b68ec81fce358678fd79c558bc8f5ea8
parent 468594 f7791b680d462524e8b8c7b640ca2f5a4a727c38
child 468596 382518bd8029480395692ca5c724c2b10cf205a1
push id35843
push usernbeleuzu@mozilla.com
push dateTue, 09 Apr 2019 22:08:13 +0000
treeherdermozilla-central@a31032a16330 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium, Callek
bugs1458385
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1458385: Repackage windows builds on linux. r=glandium,Callek This only uses cross-platform tools, so switch to running these on linux, which cuts the runtime down from ~20m to ~3m. Differential Revision: https://phabricator.services.mozilla.com/D1080
taskcluster/ci/repackage-l10n/kind.yml
taskcluster/ci/repackage-msi/kind.yml
taskcluster/ci/repackage/kind.yml
taskcluster/taskgraph/transforms/repackage.py
taskcluster/taskgraph/transforms/repackage_partner.py
testing/mozharness/configs/repackage/win32_partner.py
testing/mozharness/configs/repackage/win32_signed.py
testing/mozharness/configs/repackage/win64_partner.py
testing/mozharness/configs/repackage/win64_signed.py
testing/mozharness/scripts/repackage.py
--- a/taskcluster/ci/repackage-l10n/kind.yml
+++ b/taskcluster/ci/repackage-l10n/kind.yml
@@ -32,16 +32,19 @@ only-for-build-platforms:
     - linux-devedition-nightly/opt
     - linux64-devedition-nightly/opt
     - macosx64-devedition-nightly/opt
     - win32-devedition-nightly/opt
     - win64-devedition-nightly/opt
     - win64-aarch64-devedition-nightly/opt
 
 job-template:
+    worker-type: 'aws-provisioner-v1/gecko-{level}-b-linux'
+    worker:
+        docker-image: {"in-tree": "debian7-amd64-build"}
     mozharness:
         config:
             by-build-platform:
                 linux-.*:
                     - repackage/base.py
                     - repackage/linux32_signed.py
                 linux64-.*:
                     - repackage/base.py
@@ -72,8 +75,12 @@ job-template:
                     win64\b.*: [mar, mar-bz2, installer]
             default:
                 by-build-platform:
                     linux.*: [mar]
                     linux4\b.*: [mar]
                     macosx64\b.*: [mar, dmg]
                     win32\b.*: [mar, installer]
                     win64\b.*: [mar, installer]
+    fetches:
+        toolchain:
+            - linux64-mar-tools
+            - linux64-upx
--- a/taskcluster/ci/repackage-msi/kind.yml
+++ b/taskcluster/ci/repackage-msi/kind.yml
@@ -20,17 +20,20 @@ only-for-build-platforms:
     - win32-shippable/opt
     - win32-nightly/opt
     - win64-shippable/opt
     - win64-nightly/opt
     - win32-devedition-nightly/opt
     - win64-devedition-nightly/opt
 
 job-template:
+    worker-type: 'aws-provisioner-v1/gecko-{level}-b-win2012'
     mozharness:
+        use-magic-mh-args: false
+        use-caches: false
         config:
             by-build-platform:
                 win32\b.*:
                     - repackage/base.py
                     - repackage/win32_sfx_stub.py
                     - repackage/win32_signed.py
                 win64(?!-aarch64)\b.*:
                     - repackage/base.py
--- a/taskcluster/ci/repackage/kind.yml
+++ b/taskcluster/ci/repackage/kind.yml
@@ -42,16 +42,19 @@ only-for-build-platforms:
     - win64-aarch64-shippable/opt
     - win64-aarch64-nightly/opt
     - win64-aarch64-devedition-nightly/opt
     - win64/opt
     - win64-pgo/opt
     - win64-asan-reporter-nightly/opt
 
 job-template:
+    worker-type: 'aws-provisioner-v1/gecko-{level}-b-linux'
+    worker:
+        docker-image: {"in-tree": "debian7-amd64-build"}
     mozharness:
         config:
             by-build-platform:
                 linux\b.*:
                     - repackage/base.py
                     - repackage/linux32_signed.py
                 linux64\b.*:
                     - repackage/base.py
@@ -82,8 +85,12 @@ job-template:
                     win64\b.*: [mar, mar-bz2, installer]
             default:
                 by-build-platform:
                     linux.*: [mar]
                     linux4\b.*: [mar]
                     macosx64\b.*: [mar, dmg]
                     win32\b.*: [mar, installer]
                     win64\b.*: [mar, installer]
+    fetches:
+        toolchain:
+            - linux64-mar-tools
+            - linux64-upx
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -12,29 +12,32 @@ import copy
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import (
     optionally_keyed_by,
     resolve_keyed_by,
 )
 from taskgraph.util.taskcluster import get_artifact_prefix
-from taskgraph.util.platforms import archive_format, executable_extension, architecture
+from taskgraph.util.platforms import archive_format, architecture
 from taskgraph.util.workertypes import worker_type_implementation
 from taskgraph.transforms.job import job_description_schema
-from voluptuous import Required, Optional
+from voluptuous import Required, Optional, Extra
 
 
 packaging_description_schema = schema.extend({
     # depname is used in taskref's to identify the taskID of the signed things
     Required('depname', default='build'): basestring,
 
     # unique label to describe this repackaging task
     Optional('label'): basestring,
 
+    Optional('worker-type'): basestring,
+    Optional('worker'): object,
+
     # treeherder is allowed here to override any defaults we use for repackaging.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
     # below transforms for defaults of various values.
     Optional('treeherder'): job_description_schema['treeherder'],
 
     # If a l10n task, the corresponding locale
     Optional('locale'): basestring,
 
@@ -51,16 +54,17 @@ packaging_description_schema = schema.ex
     Optional('shipping-product'): job_description_schema['shipping-product'],
     Optional('shipping-phase'): job_description_schema['shipping-phase'],
 
     Required('package-formats'): optionally_keyed_by(
         'build-platform', 'release-type', [basestring]),
 
     # All l10n jobs use mozharness
     Required('mozharness'): {
+        Extra: object,
         # Config files passed to the mozharness script
         Required('config'): optionally_keyed_by('build-platform', [basestring]),
 
         # Additional paths to look for mozharness configs in. These should be
         # relative to the base of the source checkout
         Optional('config-paths'): [basestring],
 
         # if true, perform a checkout of a comm-central based branch inside the
@@ -71,39 +75,39 @@ packaging_description_schema = schema.ex
 
 # The configuration passed to the mozharness repackage script. This defines the
 # arguments passed to `mach repackage`
 # - `args` is interpolated by mozharness (`{package-name}`, `{installer-tag}`,
 #   `{stub-installer-tag}`, `{sfx-stub}`, `{wsx-stub}`, `{fetch-dir}`), with values
 #    from mozharness.
 # - `inputs` are passed as long-options, with the filename prefixed by
 #   `MOZ_FETCH_DIR`. The filename is interpolated by taskgraph
-#   (`{archive_format}`, `{executable_extension}`).
+#   (`{archive_format}`).
 # - `output` is passed to `--output`, with the filename prefixed by the output
 #   directory.
 PACKAGE_FORMATS = {
     'mar': {
         'args': [
             'mar',
             '--arch', '{architecture}',
         ],
         'inputs': {
             'input': 'target{archive_format}',
-            'mar': 'mar-tools/mar{executable_extension}',
+            'mar': 'mar-tools/mar',
         },
         'output': "target.complete.mar",
     },
     'mar-bz2': {
         'args': [
             'mar', "--format", "bz2",
             '--arch', '{architecture}',
         ],
         'inputs': {
             'input': 'target{archive_format}',
-            'mar': 'mar{executable_extension}',
+            'mar': 'mar',
         },
         'output': "target.bz2.complete.mar",
     },
     'msi': {
         'args': ['msi', '--wsx', '{wsx-stub}',
                  '--version', '{version_display}',
                  '--locale', '{_locale}',
                  '--arch', '{architecture}',
@@ -200,20 +204,16 @@ def make_repackage_description(config, j
 def make_job_description(config, jobs):
     for job in jobs:
         dep_job = job['primary-dependency']
         dependencies = {dep_job.attributes.get('kind'): dep_job.label}
         if len(dep_job.dependencies) > 1 and not config.kind == 'repackage-msi':
             # repackage-signing can end up with multiple deps...
             raise NotImplementedError(
                 "Can't repackage a signing task with multiple dependencies")
-        signing_dependencies = dep_job.dependencies
-        # This is so we get the build task in our dependencies to
-        # have better beetmover support.
-        dependencies.update(signing_dependencies)
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes['repackage_type'] = 'repackage'
 
         locale = attributes.get('locale', job.get('locale'))
         if locale:
             attributes['locale'] = locale
 
@@ -225,53 +225,42 @@ def make_job_description(config, jobs):
         dep_th_platform = dep_job.task.get('extra', {}).get('treeherder-platform')
         treeherder.setdefault('platform', dep_th_platform)
         treeherder.setdefault('tier', 1)
         treeherder.setdefault('kind', 'build')
 
         if config.kind == 'repackage-msi':
             treeherder['symbol'] = 'MSI({})'.format(locale or 'N')
 
-        build_task = None
         signing_task = None
         repackage_signing_task = None
         for dependency in dependencies.keys():
             if 'repackage-signing' in dependency:
                 repackage_signing_task = dependency
             elif 'signing' in dependency:
                 signing_task = dependency
-            else:
-                build_task = dependency
 
         _fetch_subst_locale = 'en-US'
         if locale:
-            # XXXCallek: todo: rewrite dependency finding
-            # Use string splice to strip out 'nightly-l10n-' .. '-<chunk>/opt'
-            # We need this additional dependency to support finding the mar binary
-            # Which is needed in order to generate a new complete.mar
-            dependencies['build'] = "build-{}/opt".format(
-                dependencies[build_task][13:dependencies[build_task].rfind('-')])
-            build_task = 'build'
             _fetch_subst_locale = locale
 
-        level = config.params['level']
+        worker_type = job['worker-type']
         build_platform = attributes['build_platform']
 
         use_stub = attributes.get('stub-installer')
 
         repackage_config = []
         package_formats = job.get('package-formats')
         if use_stub and not repackage_signing_task:
             # if repackage_signing_task doesn't exists, generate the stub installer
             package_formats += ['installer-stub']
         for format in package_formats:
             command = copy.deepcopy(PACKAGE_FORMATS[format])
             substs = {
                 'archive_format': archive_format(build_platform),
-                'executable_extension': executable_extension(build_platform),
                 '_locale': _fetch_subst_locale,
                 'architecture': architecture(build_platform),
                 'version_display': config.params['version'],
             }
             # Allow us to replace args a well, but specifying things expanded in mozharness
             # Without breaking .format and without allowing unknown through
             substs.update({name: '{{{}}}'.format(name)
                            for name in MOZHARNESS_EXPANSIONS})
@@ -293,42 +282,28 @@ def make_job_description(config, jobs):
             'job-script': 'taskcluster/scripts/builder/repackage.sh',
             'actions': ['setup', 'repackage'],
             'extra-workspace-cache-key': 'repackage',
             'extra-config': {
                 'repackage_config': repackage_config,
             },
         })
 
-        worker = {
+        worker = job.get('worker', {})
+        worker.update({
             'chain-of-trust': True,
             'max-run-time': 7200 if build_platform.startswith('win') else 3600,
             # Don't add generic artifact directory.
             'skip-artifacts': True,
-        }
+        })
 
         if locale:
             # Make sure we specify the locale-specific upload dir
             worker.setdefault('env', {}).update(LOCALE=locale)
 
-        if build_platform.startswith('win'):
-            worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
-            run['use-magic-mh-args'] = False
-            run['use-caches'] = False
-        else:
-            if build_platform.startswith(('linux', 'macosx')):
-                worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
-            else:
-                raise NotImplementedError(
-                    'Unsupported build_platform: "{}"'.format(build_platform)
-                )
-
-            run['tooltool-downloads'] = 'internal'
-            worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
-
         worker['artifacts'] = _generate_task_output_files(
             dep_job, worker_type_implementation(config.graph_config, worker_type),
             repackage_config=repackage_config,
             locale=locale,
         )
 
         description = (
             "Repackaging for locale '{locale}' for build '"
@@ -346,34 +321,34 @@ def make_job_description(config, jobs):
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
             'routes': job.get('routes', []),
             'extra': job.get('extra', {}),
             'worker': worker,
             'run': run,
-            'fetches': _generate_download_config(dep_job, build_platform, build_task,
+            'fetches': _generate_download_config(dep_job, build_platform,
                                                  signing_task, repackage_signing_task,
                                                  locale=locale,
                                                  project=config.params["project"],
                                                  existing_fetch=job.get('fetches')),
             'release-artifacts': [artifact['name'] for artifact in worker['artifacts']]
         }
 
         if build_platform.startswith('macosx'):
             task['toolchains'] = [
                 'linux64-libdmg',
                 'linux64-hfsplus',
                 'linux64-node',
             ]
         yield task
 
 
-def _generate_download_config(task, build_platform, build_task, signing_task,
+def _generate_download_config(task, build_platform, signing_task,
                               repackage_signing_task, locale=None, project=None,
                               existing_fetch=None):
     locale_path = '{}/'.format(locale) if locale else ''
     fetch = {}
     if existing_fetch:
         fetch.update(existing_fetch)
 
     if repackage_signing_task and build_platform.startswith('win'):
@@ -385,30 +360,26 @@ def _generate_download_config(task, buil
     elif build_platform.startswith('linux') or build_platform.startswith('macosx'):
         fetch.update({
             signing_task: [
                 {
                     'artifact': '{}target{}'.format(locale_path, archive_format(build_platform)),
                     'extract': False,
                 },
             ],
-            "toolchain": ["linux64-mar-tools"],
         })
     elif build_platform.startswith('win'):
         fetch.update({
             signing_task: [
                 {
                     'artifact': '{}target.zip'.format(locale_path),
                     'extract': False,
                 },
                 '{}setup.exe'.format(locale_path),
             ],
-            build_task: [
-                'host/bin/mar.exe',
-            ],
         })
 
         use_stub = task.attributes.get('stub-installer')
         if use_stub:
             fetch[signing_task].append('{}setup-stub.exe'.format(locale_path))
 
     if fetch:
         return fetch
--- a/taskcluster/taskgraph/transforms/repackage_partner.py
+++ b/taskcluster/taskgraph/transforms/repackage_partner.py
@@ -162,29 +162,18 @@ def make_job_description(config, jobs):
             'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False,
             'env': {
                 'REPACK_ID': repack_id,
             },
             # Don't add generic artifact directory.
             'skip-artifacts': True,
         }
 
-        if build_platform.startswith('win'):
-            worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
-            run['use-magic-mh-args'] = False
-        else:
-            if build_platform.startswith('macosx'):
-                worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
-            else:
-                raise NotImplementedError(
-                    'Unsupported build_platform: "{}"'.format(build_platform)
-                )
-
-            run['tooltool-downloads'] = 'internal'
-            worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
+        worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
+        worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
 
         worker['artifacts'] = _generate_task_output_files(
             dep_job, worker_type_implementation(config.graph_config, worker_type),
             repackage_config, partner=repack_id,
         )
 
         description = (
             "Repackaging for repack_id '{repack_id}' for build '"
--- a/testing/mozharness/configs/repackage/win32_partner.py
+++ b/testing/mozharness/configs/repackage/win32_partner.py
@@ -1,14 +1,14 @@
 import os
 
 platform = "win32"
 
 config = {
     "repack_id": os.environ.get("REPACK_ID"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
     'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+    'run_configure': False,
 
-    'run_configure': False,
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }
--- a/testing/mozharness/configs/repackage/win32_signed.py
+++ b/testing/mozharness/configs/repackage/win32_signed.py
@@ -1,14 +1,13 @@
 import os
 
 platform = "win32"
 
 config = {
     "locale": os.environ.get("LOCALE"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
-    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+    'run_configure': False,
 
-    'run_configure': False,
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }
--- a/testing/mozharness/configs/repackage/win64_partner.py
+++ b/testing/mozharness/configs/repackage/win64_partner.py
@@ -1,14 +1,14 @@
 import os
 
 platform = "win64"
 
 config = {
     "repack_id": os.environ.get("REPACK_ID"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
     'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+    'run_configure': False,
 
-    'run_configure': False,
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }
--- a/testing/mozharness/configs/repackage/win64_signed.py
+++ b/testing/mozharness/configs/repackage/win64_signed.py
@@ -1,14 +1,13 @@
 import os
 
 platform = "win64"
 
 config = {
     "locale": os.environ.get("LOCALE"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
-    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
+    'run_configure': False,
 
-    'run_configure': False,
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }
--- a/testing/mozharness/scripts/repackage.py
+++ b/testing/mozharness/scripts/repackage.py
@@ -88,16 +88,17 @@ class Repackage(BaseScript):
                 ])
             command.extend([
                 '--output', os.path.join(dirs['abs_output_dir'], repack_config['output']),
             ])
             self.run_command(
                 command=command,
                 cwd=dirs['abs_mozilla_dir'],
                 halt_on_failure=True,
+                env=self.query_env(),
             )
 
     def _run_tooltool(self):
         config = self.config
         dirs = self.query_abs_dirs()
         toolchains = os.environ.get('MOZ_TOOLCHAINS')
         manifest_src = os.environ.get('TOOLTOOL_MANIFEST')
         if not manifest_src: