Bug 1458385: Package mar and mbsdiff as a toolchain; r=glandium
☠☠ backed out by 4e2ea1a75e87 ☠ ☠
authorTom Prince <mozilla@hocat.ca>
Wed, 27 Mar 2019 22:30:44 +0000
changeset 528351 9a5f255979c2f1f251d0d96a93e648cd98ca6b3f
parent 528350 15c4fae5d559ec6be2b8409ea7fb76af05d90962
child 528352 99fa5539c2bda621abac134300428129969b2cdf
push id2082
push userffxbld-merge
push dateMon, 01 Jul 2019 08:34:18 +0000
treeherdermozilla-release@2fb19d0466d2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1458385
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1458385: Package mar and mbsdiff as a toolchain; r=glandium Differential Revision: https://phabricator.services.mozilla.com/D24229
build/sparse-profiles/taskgraph
mfbt/moz.build
other-licenses/bsdiff/moz.build
taskcluster/ci/toolchain/misc.yml
taskcluster/scripts/misc/build-mar-tools.sh
taskcluster/taskgraph/transforms/repackage.py
tools/update-packaging/app.mozbuild
tools/update-packaging/moz.configure
--- a/build/sparse-profiles/taskgraph
+++ b/build/sparse-profiles/taskgraph
@@ -51,8 +51,15 @@ path:mobile/android/config/
 glob:**/*.gradle
 
 # for action-task building
 path:.taskcluster.yml
 path:.cron.yml
 
 # for the wrench-deps toolchain task
 path:gfx/wr/Cargo.lock
+
+# for the mar-tools toolchain task
+path:mfbt/
+path:modules/libmar/
+path:other-licenses/bsdiff/
+path:other-licenses/nsis/Contrib/CityHash/cityhash/
+path:toolkit/mozapps/update/updater
--- a/mfbt/moz.build
+++ b/mfbt/moz.build
@@ -149,17 +149,17 @@ UNIFIED_SOURCES += [
     'Poison.cpp',
     'RandomNum.cpp',
     'SHA1.cpp',
     'TaggedAnonymousMemory.cpp',
     'Unused.cpp',
     'Utf8.cpp',
 ]
 
-if CONFIG['MOZ_BUILD_APP'] != 'tools/crashreporter':
+if CONFIG['MOZ_BUILD_APP'] not in ('tools/update-packaging', 'tools/crashreporter'):
     # RecordReplay.cpp uses files from js/
     UNIFIED_SOURCES += [
         'RecordReplay.cpp',
     ]
 
     # Building MFBT tests adds a large overhead when building
     # tools/crashreporter.
     TEST_DIRS += ['tests']
--- a/other-licenses/bsdiff/moz.build
+++ b/other-licenses/bsdiff/moz.build
@@ -13,9 +13,10 @@ HostProgram('mbsdiff')
 if CONFIG['HOST_OS_ARCH'] == 'WINNT':
     HOST_OS_LIBS += [
         'ws2_32',
     ]
     USE_STATIC_LIBS = True
 
 LOCAL_INCLUDES += [
     '/toolkit/mozapps/update/updater',
+    '/toolkit/mozapps/update/updater/bspatch',
 ]
--- a/taskcluster/ci/toolchain/misc.yml
+++ b/taskcluster/ci/toolchain/misc.yml
@@ -65,16 +65,33 @@ linux64-hfsplus:
 linux64-libdmg:
     description: "libdmg-hfsplus toolchain build"
     treeherder:
         symbol: TL(libdmg-hfs+)
     run:
         script: build-libdmg-hfsplus.sh
         toolchain-artifact: public/build/dmg.tar.xz
 
+linux64-mar-tools:
+    description: "mar-tools toolchain build"
+    treeherder:
+        symbol: TL(mar-tools)
+    toolchains:
+        - linux64-clang-7
+        - linux64-binutils
+    run:
+        script: build-mar-tools.sh
+        sparse-profile: null
+        toolchain-artifact: public/build/mar-tools.tar.xz
+        resources:
+            - modules/libmar/
+            - other-licenses/nsis/Contrib/CityHash/cityhash/
+            - other-licenses/bsdiff/
+            - toolkit/mozapps/update/updater/bspatch/
+
 linux64-tup:
     description: "tup toolchain build"
     treeherder:
         symbol: TL(tup)
     worker:
         max-run-time: 3600
     run:
         script: build-tup-linux.sh
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-mar-tools.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building mar and mbsdiff
+
+WORKSPACE=$HOME/workspace
+UPLOAD_DIR=$HOME/artifacts
+COMPRESS_EXT=xz
+
+cd $WORKSPACE/build/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+
+export MOZ_OBJDIR=obj-mar
+
+echo ac_add_options --enable-project=tools/update-packaging > .mozconfig
+
+TOOLCHAINS="binutils clang"
+
+for t in $TOOLCHAINS; do
+    PATH="$WORKSPACE/build/src/$t/bin:$PATH"
+done
+
+./mach build -v
+
+mkdir mar-tools
+cp $MOZ_OBJDIR/dist/host/bin/{mar,mbsdiff} mar-tools/
+
+tar -acf mar-tools.tar.$COMPRESS_EXT mar-tools/
+mkdir -p $UPLOAD_DIR
+cp mar-tools.tar.$COMPRESS_EXT $UPLOAD_DIR
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -82,17 +82,17 @@ packaging_description_schema = schema.ex
 PACKAGE_FORMATS = {
     'mar': {
         'args': [
             'mar',
             '--arch', '{architecture}',
         ],
         'inputs': {
             'input': 'target{archive_format}',
-            'mar': 'mar{executable_extension}',
+            'mar': 'mar-tools/mar{executable_extension}',
         },
         'output': "target.complete.mar",
     },
     'mar-bz2': {
         'args': [
             'mar', "--format", "bz2",
             '--arch', '{architecture}',
         ],
@@ -385,19 +385,17 @@ def _generate_download_config(task, buil
     elif build_platform.startswith('linux') or build_platform.startswith('macosx'):
         fetch.update({
             signing_task: [
                 {
                     'artifact': '{}target{}'.format(locale_path, archive_format(build_platform)),
                     'extract': False,
                 },
             ],
-            build_task: [
-                'host/bin/mar',
-            ],
+            "toolchain": ["linux64-mar-tools"],
         })
     elif build_platform.startswith('win'):
         fetch.update({
             signing_task: [
                 {
                     'artifact': '{}target.zip'.format(locale_path),
                     'extract': False,
                 },
new file mode 100644
--- /dev/null
+++ b/tools/update-packaging/app.mozbuild
@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += [
+    '/modules/libmar/src',
+    '/modules/libmar/tool',
+    '/other-licenses/bsdiff',
+]
new file mode 100644