Bug 1458385: Package mar and mbsdiff as a toolchain; r=glandium
authorTom Prince <mozilla@hocat.ca>
Sat, 06 Apr 2019 05:49:31 +0000
changeset 468592 3b10487587c399057f13465266e1130c609332a9
parent 468591 20c3796517dababd5f650efcfeaaa361f1474b95
child 468593 cc06a7beb3d14069f406f7846b55d8e2f38e39ed
push id35843
push usernbeleuzu@mozilla.com
push dateTue, 09 Apr 2019 22:08:13 +0000
treeherdermozilla-central@a31032a16330 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1458385
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1458385: Package mar and mbsdiff as a toolchain; r=glandium Differential Revision: https://phabricator.services.mozilla.com/D24229
build/sparse-profiles/taskgraph
mfbt/moz.build
other-licenses/bsdiff/moz.build
taskcluster/ci/toolchain/misc.yml
taskcluster/scripts/misc/build-mar-tools.sh
taskcluster/taskgraph/transforms/repackage.py
tools/update-packaging/app.mozbuild
tools/update-packaging/moz.configure
--- a/build/sparse-profiles/taskgraph
+++ b/build/sparse-profiles/taskgraph
@@ -51,8 +51,15 @@ path:mobile/android/config/
 glob:**/*.gradle
 
 # for action-task building
 path:.taskcluster.yml
 path:.cron.yml
 
 # for the wrench-deps toolchain task
 path:gfx/wr/Cargo.lock
+
+# for the mar-tools toolchain task
+path:mfbt/
+path:modules/libmar/
+path:other-licenses/bsdiff/
+path:other-licenses/nsis/Contrib/CityHash/cityhash/
+path:toolkit/mozapps/update/updater
--- a/mfbt/moz.build
+++ b/mfbt/moz.build
@@ -150,17 +150,17 @@ UNIFIED_SOURCES += [
     'Poison.cpp',
     'RandomNum.cpp',
     'SHA1.cpp',
     'TaggedAnonymousMemory.cpp',
     'Unused.cpp',
     'Utf8.cpp',
 ]
 
-if CONFIG['MOZ_BUILD_APP'] not in ('memory', 'tools/crashreporter'):
+if CONFIG['MOZ_BUILD_APP'] not in ('memory', 'tools/crashreporter', 'tools/update-packaging'):
     # RecordReplay.cpp uses files from js/
     UNIFIED_SOURCES += [
         'RecordReplay.cpp',
     ]
 
     # Building MFBT tests adds a large overhead when building
     # tools/crashreporter.
     TEST_DIRS += ['tests']
--- a/other-licenses/bsdiff/moz.build
+++ b/other-licenses/bsdiff/moz.build
@@ -13,9 +13,10 @@ HostProgram('mbsdiff')
 if CONFIG['HOST_OS_ARCH'] == 'WINNT':
     HOST_OS_LIBS += [
         'ws2_32',
     ]
     USE_STATIC_LIBS = True
 
 LOCAL_INCLUDES += [
     '/toolkit/mozapps/update/updater',
+    '/toolkit/mozapps/update/updater/bspatch',
 ]
--- a/taskcluster/ci/toolchain/misc.yml
+++ b/taskcluster/ci/toolchain/misc.yml
@@ -65,16 +65,33 @@ linux64-hfsplus:
 linux64-libdmg:
     description: "libdmg-hfsplus toolchain build"
     treeherder:
         symbol: TL(libdmg-hfs+)
     run:
         script: build-libdmg-hfsplus.sh
         toolchain-artifact: public/build/dmg.tar.xz
 
+linux64-mar-tools:
+    description: "mar-tools toolchain build"
+    treeherder:
+        symbol: TL(mar-tools)
+    toolchains:
+        - linux64-clang-7
+        - linux64-binutils
+    run:
+        script: build-mar-tools.sh
+        sparse-profile: null
+        toolchain-artifact: public/build/mar-tools.tar.xz
+        resources:
+            - modules/libmar/
+            - other-licenses/nsis/Contrib/CityHash/cityhash/
+            - other-licenses/bsdiff/
+            - toolkit/mozapps/update/updater/bspatch/
+
 linux64-tup:
     description: "tup toolchain build"
     treeherder:
         symbol: TL(tup)
     worker:
         max-run-time: 3600
     run:
         script: build-tup-linux.sh
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-mar-tools.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building mar and mbsdiff
+
+WORKSPACE=$HOME/workspace
+UPLOAD_DIR=$HOME/artifacts
+COMPRESS_EXT=xz
+
+cd $WORKSPACE/build/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+
+export MOZ_OBJDIR=obj-mar
+
+echo ac_add_options --enable-project=tools/update-packaging > .mozconfig
+
+TOOLCHAINS="binutils clang"
+
+for t in $TOOLCHAINS; do
+    PATH="$WORKSPACE/build/src/$t/bin:$PATH"
+done
+
+./mach build -v
+
+mkdir mar-tools
+cp $MOZ_OBJDIR/dist/host/bin/{mar,mbsdiff} mar-tools/
+
+tar -acf mar-tools.tar.$COMPRESS_EXT mar-tools/
+mkdir -p $UPLOAD_DIR
+cp mar-tools.tar.$COMPRESS_EXT $UPLOAD_DIR
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -82,17 +82,17 @@ packaging_description_schema = schema.ex
 PACKAGE_FORMATS = {
     'mar': {
         'args': [
             'mar',
             '--arch', '{architecture}',
         ],
         'inputs': {
             'input': 'target{archive_format}',
-            'mar': 'mar{executable_extension}',
+            'mar': 'mar-tools/mar{executable_extension}',
         },
         'output': "target.complete.mar",
     },
     'mar-bz2': {
         'args': [
             'mar', "--format", "bz2",
             '--arch', '{architecture}',
         ],
@@ -385,19 +385,17 @@ def _generate_download_config(task, buil
     elif build_platform.startswith('linux') or build_platform.startswith('macosx'):
         fetch.update({
             signing_task: [
                 {
                     'artifact': '{}target{}'.format(locale_path, archive_format(build_platform)),
                     'extract': False,
                 },
             ],
-            build_task: [
-                'host/bin/mar',
-            ],
+            "toolchain": ["linux64-mar-tools"],
         })
     elif build_platform.startswith('win'):
         fetch.update({
             signing_task: [
                 {
                     'artifact': '{}target.zip'.format(locale_path),
                     'extract': False,
                 },
new file mode 100644
--- /dev/null
+++ b/tools/update-packaging/app.mozbuild
@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += [
+    '/modules/libmar/src',
+    '/modules/libmar/tool',
+    '/other-licenses/bsdiff',
+]
new file mode 100644