Bug 1427145 - Use toolchain artifacts instead of tooltool packages for osx (cross) repackages. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 27 Dec 2017 07:02:21 +0900
changeset 397507 2d6c872f416ad1683e6fb952b07840b92ec0afff
parent 397506 bb6003aa6966827c609172fa311a2a15879cb41b
child 397508 2e671075bdf41bddaeb1658180926723e5fa64b6
child 397511 30386b610adba825c1d5298389d933a61ef6a750
push id33166
push usershindli@mozilla.com
push dateSat, 30 Dec 2017 09:53:20 +0000
treeherdermozilla-central@2e671075bdf4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1427145
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1427145 - Use toolchain artifacts instead of tooltool packages for osx (cross) repackages. r=gps OSX (cross) repackages are currently using a tooltool manifest to get libdmg and hfsplus. Change those jobs to use the toolchain artifacts instead. At the same time, modify the repackage mozharness script's _run_tooltool so that it doesn't fail with MOZ_TOOLCHAINS being set but without a tooltool_manifest_src, matching the similar function in buildbase.py.
browser/config/tooltool-manifests/macosx64/cross-l10n.manifest
taskcluster/ci/l10n/kind.yml
taskcluster/ci/nightly-l10n/kind.yml
taskcluster/ci/repackage-l10n/kind.yml
taskcluster/ci/repackage/kind.yml
taskcluster/taskgraph/transforms/repackage.py
testing/mozharness/configs/repackage/osx_signed.py
testing/mozharness/configs/single_locale/tc_macosx64.py
testing/mozharness/scripts/repackage.py
deleted file mode 100644
--- a/browser/config/tooltool-manifests/macosx64/cross-l10n.manifest
+++ /dev/null
@@ -1,19 +0,0 @@
-[
-  {
-    "version": "https://github.com/mozilla/libdmg-hfsplus rev ba04b00435a0853f1499d751617177828ee8ec00",
-    "size": 57060,
-    "visibility": "public",
-    "digest": "0c96f0d3ace71c4110abec6f7ead013600b0a73c89465d840276090d849279e555d977fb2aa6bbabb7891d7191fc8cc8a4e8242be888114be52346b77a512fcc",
-    "algorithm": "sha512",
-    "unpack": true,
-    "filename": "dmg.tar.xz"
-  },
-  {
-    "size": 281576,
-    "visibility": "public",
-    "digest": "71616564533d138fb12f08e761c2638d054814fdf9c9439638ec57b201e100445c364d73d8d7a4f0e3b784898d5fe6264e8242863fc5ac40163f1791468bbc46",
-    "algorithm": "sha512",
-    "filename": "hfsplus-tools.tar.xz",
-    "unpack": true
-  }
-]
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -48,16 +48,19 @@ job-template:
          win.*: null
    secrets:
       by-build-platform:
          default: false
          android-api-16-l10n: true
    toolchains:
       by-build-platform:
          default: []
+         macosx64:
+            - linux64-libdmg
+            - linux64-hfsplus
          android-api-16-l10n:
             - android-gradle-dependencies
             - android-sdk-linux
             - proguard-jar
    tooltool:
       by-build-platform:
          default: public
          android-api-16-l10n: internal
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -58,16 +58,19 @@ job-template:
          win.*: null
    secrets:
       by-build-platform:
          default: false
          android-api-16-nightly: true
    toolchains:
       by-build-platform:
          default: []
+         macosx64.*:
+            - linux64-libdmg
+            - linux64-hfsplus
          android-api-16-nightly:
             - android-gradle-dependencies
             - android-sdk-linux
             - proguard-jar
    tooltool:
       by-build-platform:
          default: public
          android-api-16-nightly: internal
--- a/taskcluster/ci/repackage-l10n/kind.yml
+++ b/taskcluster/ci/repackage-l10n/kind.yml
@@ -3,21 +3,23 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
    - taskgraph.transforms.repackage_l10n:transforms
    - taskgraph.transforms.name_sanity:transforms
    - taskgraph.transforms.repackage:transforms
+   - taskgraph.transforms.use_toolchains:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - nightly-l10n-signing
+   - toolchain
 
 only-for-build-platforms:
    - linux-nightly/opt
    - linux64-nightly/opt
    - macosx64-nightly/opt
    - win32-nightly/opt
    - win64-nightly/opt
    - linux-devedition-nightly/opt
--- a/taskcluster/ci/repackage/kind.yml
+++ b/taskcluster/ci/repackage/kind.yml
@@ -2,22 +2,24 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
     - taskgraph.transforms.name_sanity:transforms
     - taskgraph.transforms.repackage:transforms
+    - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.repackage_routes:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 kind-dependencies:
     - build-signing
+    - toolchain
 
 only-for-build-platforms:
     - linux-nightly/opt
     - linux-devedition-nightly/opt
     - linux64-nightly/opt
     - linux64-devedition-nightly/opt
     - macosx64-nightly/opt
     - macosx64-devedition-nightly/opt
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -182,16 +182,22 @@ def make_job_description(config, jobs):
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
             'routes': job.get('routes', []),
             'extra': job.get('extra', {}),
             'worker': worker,
             'run': run,
         }
+
+        if build_platform.startswith('macosx'):
+            task['toolchains'] = [
+                'linux64-libdmg',
+                'linux64-hfsplus',
+            ]
         yield task
 
 
 def _generate_task_mozharness_config(build_platform):
     if build_platform.startswith('macosx'):
         return ['repackage/osx_signed.py']
     else:
         bits = 32 if '32' in build_platform else 64
--- a/testing/mozharness/configs/repackage/osx_signed.py
+++ b/testing/mozharness/configs/repackage/osx_signed.py
@@ -19,13 +19,12 @@ config = {
     ], [
         "mar",
         "-i", "{abs_work_dir}/inputs/target.tar.gz",
         "--mar", "{abs_work_dir}/inputs/mar",
         "-o", "{output_home}/target.complete.mar"
     ]],
 
     # ToolTool
-    "tooltool_manifest_src": 'browser/config/tooltool-manifests/macosx64/cross-l10n.manifest',
     "tooltool_url": 'http://relengapi/tooltool/',
     'tooltool_script': ["/builds/tooltool.py"],
     'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
 }
--- a/testing/mozharness/configs/single_locale/tc_macosx64.py
+++ b/testing/mozharness/configs/single_locale/tc_macosx64.py
@@ -16,13 +16,12 @@ config = {
         'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
     },
     "upload_env": {
         'UPLOAD_HOST': 'localhost',
         'UPLOAD_PATH': '/builds/worker/artifacts/',
     },
 
     "tooltool_url": 'http://relengapi/tooltool/',
-    'tooltool_manifest_src': "browser/config/tooltool-manifests/macosx64/cross-l10n.manifest",
     "mozilla_dir": "src/",
     "simple_name_move": True,
 }
 
--- a/testing/mozharness/scripts/repackage.py
+++ b/testing/mozharness/scripts/repackage.py
@@ -90,35 +90,36 @@ class Repackage(BaseScript):
         dirs = self.query_abs_dirs()
         toolchains = os.environ.get('MOZ_TOOLCHAINS')
         manifest_src = os.environ.get('TOOLTOOL_MANIFEST')
         if not manifest_src:
             manifest_src = config.get('tooltool_manifest_src')
         if not manifest_src and not toolchains:
             return
 
-        tooltool_manifest_path = os.path.join(dirs['abs_mozilla_dir'],
-                                              manifest_src)
         cmd = [
             sys.executable, '-u',
             os.path.join(dirs['abs_mozilla_dir'], 'mach'),
             'artifact',
             'toolchain',
             '-v',
             '--retry', '4',
-            '--tooltool-manifest',
-            tooltool_manifest_path,
             '--artifact-manifest',
             os.path.join(dirs['abs_mozilla_dir'], 'toolchains.json'),
-            '--tooltool-url',
-            config['tooltool_url'],
         ]
-        auth_file = self._get_tooltool_auth_file()
-        if auth_file:
-            cmd.extend(['--authentication-file', auth_file])
+        if manifest_src:
+            cmd.extend([
+                '--tooltool-manifest',
+                os.path.join(dirs['abs_mozilla_dir'], manifest_src),
+                '--tooltool-url',
+                config['tooltool_url'],
+            ])
+            auth_file = self._get_tooltool_auth_file()
+            if auth_file:
+                cmd.extend(['--authentication-file', auth_file])
         cache = config.get('tooltool_cache')
         if cache:
             cmd.extend(['--cache-dir', cache])
         if toolchains:
             cmd.extend(toolchains.split())
         self.info(str(cmd))
         self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)