Bug 1405413 - Migrate Gradle dependency fetching to toolchain dependencies. r=dustin draft
authorNick Alexander <nalexander@mozilla.com>
Tue, 03 Oct 2017 13:09:49 -0700
changeset 678600 6e4397f157108081ad90de7bad6432c583f9ff5f
parent 678599 97e3f1b1ed36c00b9033fc92ec15e7e50a897820
child 678601 98440d11720b2f0b67d7ad512f3ba9ac5d84fd9a
push id83975
push usernalexander@mozilla.com
push dateWed, 11 Oct 2017 16:10:55 +0000
reviewersdustin
bugs1405413
milestone58.0a1
Bug 1405413 - Migrate Gradle dependency fetching to toolchain dependencies. r=dustin MozReview-Commit-ID: I9i29g6C2CS
mobile/android/config/mozconfigs/android-api-16-frontend/nightly
mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
mobile/android/config/mozconfigs/android-api-16-gradle/nightly
mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
mobile/android/config/tooltool-manifests/android-x86/releng.manifest
mobile/android/config/tooltool-manifests/android/releng.manifest
taskcluster/ci/build/android-stuff.yml
taskcluster/ci/build/android.yml
taskcluster/ci/toolchain/linux.yml
taskcluster/scripts/builder/build-android-dependencies.sh
taskcluster/scripts/builder/build-android-dependencies/after.sh
taskcluster/scripts/builder/build-android-dependencies/before.sh
taskcluster/scripts/builder/build-android-dependencies/nexus.xml
taskcluster/scripts/misc/android-gradle-dependencies.sh
taskcluster/scripts/misc/android-gradle-dependencies/after.sh
taskcluster/scripts/misc/android-gradle-dependencies/before.sh
taskcluster/scripts/misc/android-gradle-dependencies/nexus.xml
--- a/mobile/android/config/mozconfigs/android-api-16-frontend/nightly
+++ b/mobile/android/config/mozconfigs/android-api-16-frontend/nightly
@@ -9,18 +9,18 @@ MOZ_AUTOMATION_UPLOAD=0
 MOZ_AUTOMATION_UPLOAD_SYMBOLS=0
 MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0
 
 NO_CACHE=1
 NO_NDK=1
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common"
 
-ac_add_options --with-gradle="$topsrcdir/gradle-dist/bin/gradle"
-export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/jcenter","file://$topsrcdir/google"
+ac_add_options --with-gradle="$topsrcdir/android-gradle-dependencies/gradle-dist/bin/gradle"
+export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/android-gradle-dependencies/jcenter","file://$topsrcdir/android-gradle-dependencies/google"
 
 unset HOST_CC
 unset HOST_CXX
 unset RUSTC
 unset CARGO
 
 ac_add_options --disable-compile-environment
 ac_add_options --disable-tests
--- a/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
+++ b/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
@@ -48,8 +48,17 @@ STRIP_FLAGS="--strip-debug"
 
 export MOZILLA_OFFICIAL=1
 export MOZ_TELEMETRY_REPORTING=1
 export MOZ_ANDROID_POCKET=1
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common.override"
 
 # End ../android-api-16-frontend/nightly.
+
+# Disable Keyfile Loading (and checks) since dependency fetching doesn't need these keys.
+# This overrides the settings in the common android mozconfig
+ac_add_options --without-mozilla-api-keyfile
+ac_add_options --without-google-api-keyfile
+# We need dummy Keyfiles in order to enable features we care about.
+ac_add_options --with-adjust-sdk-keyfile="$topsrcdir/mobile/android/base/adjust-sdk-sandbox.token"
+ac_add_options --with-leanplum-sdk-keyfile="$topsrcdir/mobile/android/base/leanplum-sdk-sandbox.token"
+ac_add_options --with-pocket-api-keyfile="$topsrcdir/mobile/android/base/pocket-api-sandbox.token"
--- a/mobile/android/config/mozconfigs/android-api-16-gradle/nightly
+++ b/mobile/android/config/mozconfigs/android-api-16-gradle/nightly
@@ -1,8 +1,8 @@
 . "$topsrcdir/mobile/android/config/mozconfigs/common"
 
 . "$topsrcdir/mobile/android/config/mozconfigs/android-api-16/nightly"
 
-ac_add_options --with-gradle="$topsrcdir/gradle-dist/bin/gradle"
-export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/jcenter","file://$topsrcdir/google"
+ac_add_options --with-gradle="$topsrcdir/android-gradle-dependencies/gradle-dist/bin/gradle"
+export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/android-gradle-dependencies/jcenter","file://$topsrcdir/android-gradle-dependencies/google"
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common.override"
--- a/mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
@@ -1,34 +1,2 @@
 [
-  {
-    "size": 37130176,
-    "visibility": "internal",
-    "digest": "254eecc06eea407c098417c1dbedf89f59883cdde7e173b0473fa5b34719c967830afa11189c3a9f69dc3f3b9e5fb43469434340e1f8feb378609af965783b48",
-    "algorithm": "sha512",
-    "filename": "java_home.tar.xz",
-    "unpack": true
-  },
-  {
-    "size": 7797928,
-    "visibility": "public",
-    "digest": "7bb5c7b3586179a7805ae60b025b7856cb8dbed66da1b7962bc446d94ea6fa45ed55ebfe065d2d82690c68bf71bdab613e97f53da79b85e3185955461f751256",
-    "algorithm": "sha512",
-    "filename": "google.tar.xz",
-    "unpack": true
-  },
-  {
-    "size": 52693504,
-    "visibility": "public",
-    "digest": "d8c0d91ff4039dd8a22e34634c0f72b21b1c272d60987f20a709fbfd919ba62eab4c3bddbaf96af1c6caf85320b4bf0676589071fe3b31724c51782a55da83e0",
-    "algorithm": "sha512",
-    "filename": "jcenter.tar.xz",
-    "unpack": true
-  },
-  {
-    "algorithm": "sha512",
-    "visibility": "public",
-    "filename": "gradle-dist.tar.xz",
-    "unpack": true,
-    "digest": "e3cfe7f8259ad97722243d4e873d5a05c014bfc24d637427f89d804bf5073290229c778ea303142cf06c2dc79e0492f23521f57d3a73825f55b8db587317646f",
-    "size": 51753660
-  }
 ]
--- a/mobile/android/config/tooltool-manifests/android-x86/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-x86/releng.manifest
@@ -10,42 +10,10 @@
   },
   {
     "size": 6856444,
     "visibility": "public",
     "unpack": true,
     "digest": "d68dd7d31b0153095ecf5cde5837fb1f95dc6e3f799d496fb764f7afeb9c6095c332467177c3aa54d3749b1901e0d6fa84c42162526e764e8a9d2196a0189861",
     "algorithm": "sha512",
     "filename": "jsshell.tar.xz"
-  },
-  {
-    "size": 7797928,
-    "visibility": "public",
-    "digest": "7bb5c7b3586179a7805ae60b025b7856cb8dbed66da1b7962bc446d94ea6fa45ed55ebfe065d2d82690c68bf71bdab613e97f53da79b85e3185955461f751256",
-    "algorithm": "sha512",
-    "filename": "google.tar.xz",
-    "unpack": true
-  },
-  {
-    "size": 52693504,
-    "visibility": "public",
-    "digest": "d8c0d91ff4039dd8a22e34634c0f72b21b1c272d60987f20a709fbfd919ba62eab4c3bddbaf96af1c6caf85320b4bf0676589071fe3b31724c51782a55da83e0",
-    "algorithm": "sha512",
-    "filename": "jcenter.tar.xz",
-    "unpack": true
-  },
-  {
-    "algorithm": "sha512",
-    "visibility": "public",
-    "filename": "gradle-dist.tar.xz",
-    "unpack": true,
-    "digest": "e3cfe7f8259ad97722243d4e873d5a05c014bfc24d637427f89d804bf5073290229c778ea303142cf06c2dc79e0492f23521f57d3a73825f55b8db587317646f",
-    "size": 51753660
-  },
-  {
-    "size": 37130176,
-    "visibility": "internal",
-    "digest": "254eecc06eea407c098417c1dbedf89f59883cdde7e173b0473fa5b34719c967830afa11189c3a9f69dc3f3b9e5fb43469434340e1f8feb378609af965783b48",
-    "algorithm": "sha512",
-    "filename": "java_home.tar.xz",
-    "unpack": true
   }
 ]
--- a/mobile/android/config/tooltool-manifests/android/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android/releng.manifest
@@ -10,42 +10,10 @@
   },
   {
     "size": 6856444,
     "visibility": "public",
     "unpack": true,
     "digest": "d68dd7d31b0153095ecf5cde5837fb1f95dc6e3f799d496fb764f7afeb9c6095c332467177c3aa54d3749b1901e0d6fa84c42162526e764e8a9d2196a0189861",
     "algorithm": "sha512",
     "filename": "jsshell.tar.xz"
-  },
-  {
-    "size": 37130176,
-    "visibility": "internal",
-    "digest": "254eecc06eea407c098417c1dbedf89f59883cdde7e173b0473fa5b34719c967830afa11189c3a9f69dc3f3b9e5fb43469434340e1f8feb378609af965783b48",
-    "algorithm": "sha512",
-    "filename": "java_home.tar.xz",
-    "unpack": true
-  },
-  {
-    "size": 7797928,
-    "visibility": "public",
-    "digest": "7bb5c7b3586179a7805ae60b025b7856cb8dbed66da1b7962bc446d94ea6fa45ed55ebfe065d2d82690c68bf71bdab613e97f53da79b85e3185955461f751256",
-    "algorithm": "sha512",
-    "filename": "google.tar.xz",
-    "unpack": true
-  },
-  {
-    "size": 52693504,
-    "visibility": "public",
-    "digest": "d8c0d91ff4039dd8a22e34634c0f72b21b1c272d60987f20a709fbfd919ba62eab4c3bddbaf96af1c6caf85320b4bf0676589071fe3b31724c51782a55da83e0",
-    "algorithm": "sha512",
-    "filename": "jcenter.tar.xz",
-    "unpack": true
-  },
-  {
-    "algorithm": "sha512",
-    "visibility": "public",
-    "filename": "gradle-dist.tar.xz",
-    "unpack": true,
-    "digest": "e3cfe7f8259ad97722243d4e873d5a05c014bfc24d637427f89d804bf5073290229c778ea303142cf06c2dc79e0492f23521f57d3a73825f55b8db587317646f",
-    "size": 51753660
   }
 ]
--- a/taskcluster/ci/build/android-stuff.yml
+++ b/taskcluster/ci/build/android-stuff.yml
@@ -1,53 +1,8 @@
-android-dependencies/opt:
-    description: "Android armv7 api-16+ gradle dependencies"
-    index:
-        product: mobile
-        job-name: android-dependencies
-    treeherder:
-        platform: android-4-0-armv7-api16/opt
-        kind: build
-        tier: 2
-        symbol: tc(Deps)
-    worker-type: aws-provisioner-v1/gecko-{level}-b-android
-    worker:
-        docker-image: {in-tree: android-build}
-        env:
-            GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online"
-            PERFHERDER_EXTRA_OPTIONS: android-dependencies
-            TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest"
-            WORKSPACE: "/builds/worker/workspace"
-        artifacts:
-          - name: public/build
-            path: /builds/worker/artifacts/
-            type: directory
-        max-run-time: 36000
-    run:
-        using: mozharness
-        actions: [get-secrets build multi-l10n update]
-        config:
-            - builds/releng_base_android_64_builds.py
-            - disable_signing.py
-            - platform_supports_post_upload_to_latest.py
-        script: "mozharness/scripts/fx_desktop_build.py"
-        secrets: true
-        custom-build-variant-cfg: api-16-gradle-dependencies
-        tooltool-downloads: internal
-        job-script: taskcluster/scripts/builder/build-android-dependencies.sh
-    toolchains:
-        - android-sdk-linux
-        - proguard-jar
-    optimization:
-        skip-unless-changed:
-          - "mobile/android/config/**"
-          - "testing/mozharness/configs/builds/releng_sub_android_configs/*gradle_dependencies.py"
-          - "**/*.gradle"
-          - "taskcluster/docker/android-build/**"
-
 android-test/opt:
     description: "Android armv7 unit tests"
     index:
         product: mobile
         job-name: android-test
     treeherder:
         platform: android-4-0-armv7-api16/opt
         kind: build
@@ -75,16 +30,17 @@ android-test/opt:
             - builds/releng_base_android_64_builds.py
             - disable_signing.py
             - platform_supports_post_upload_to_latest.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: android-test
         tooltool-downloads: internal
     toolchains:
+        - android-gradle-dependencies
         - android-sdk-linux
         - proguard-jar
     optimization:
         skip-unless-changed:
           - "mobile/android/base/**"
           - "mobile/android/config/**"
           - "mobile/android/tests/background/junit4/**"
           - "**/*.gradle"
@@ -136,16 +92,17 @@ android-lint/opt:
             - builds/releng_base_android_64_builds.py
             - disable_signing.py
             - platform_supports_post_upload_to_latest.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: android-lint
         tooltool-downloads: internal
     toolchains:
+        - android-gradle-dependencies
         - android-sdk-linux
         - proguard-jar
     optimization:
         skip-unless-changed:
           - "mobile/android/**/*.java"
           - "mobile/android/**/*.jpeg"
           - "mobile/android/**/*.jpg"
           - "mobile/android/**/*.png"
@@ -191,16 +148,17 @@ android-checkstyle/opt:
             - builds/releng_base_android_64_builds.py
             - disable_signing.py
             - platform_supports_post_upload_to_latest.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: android-checkstyle
         tooltool-downloads: internal
     toolchains:
+        - android-gradle-dependencies
         - android-sdk-linux
         - proguard-jar
     optimization:
         skip-unless-changed:
           - "mobile/android/**/checkstyle.xml"
           - "mobile/android/**/*.java"
           - "mobile/android/**/Makefile.in"
           - "mobile/android/config/**"
@@ -248,16 +206,17 @@ android-findbugs/opt:
             - builds/releng_base_android_64_builds.py
             - disable_signing.py
             - platform_supports_post_upload_to_latest.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: android-findbugs
         tooltool-downloads: internal
     toolchains:
+        - android-gradle-dependencies
         - android-sdk-linux
         - proguard-jar
     optimization:
         skip-unless-changed:
           - "mobile/android/**/*.java"
           - "mobile/android/**/Makefile.in"
           - "mobile/android/config/**"
           - "mobile/android/**/moz.build"
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -330,16 +330,17 @@ android-api-16-gradle/opt:
             - builds/releng_base_android_64_builds.py
             - disable_signing.py
             - platform_supports_post_upload_to_latest.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: api-16-gradle
         tooltool-downloads: internal
     toolchains:
+        - android-gradle-dependencies
         - android-sdk-linux
         - linux64-rust-android
         - linux64-sccache
         - proguard-jar
 
 android-aarch64/opt:
     description: "Android 5.0 AArch64 Opt"
     index:
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -221,16 +221,47 @@ linux64-android-sdk-linux-repack:
         using: toolchain-script
         script: repack-android-sdk-linux.sh
         tc-vcs: false
         resources:
             - 'python/mozboot/**/*android*'
         toolchain-artifact: project/gecko/android-sdk/android-sdk-linux.tar.xz
         toolchain-alias: android-sdk-linux
 
+linux64-android-gradle-dependencies:
+    description: "Android Gradle dependencies toolchain task"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(gradle-dependencies)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        docker-image: {in-tree: android-build}
+        env:
+            GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online"
+        max-run-time: 36000
+    run:
+        using: toolchain-script
+        script: android-gradle-dependencies.sh
+
+        tc-vcs: false
+        resources:
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+            - 'taskcluster/scripts/misc/android-gradle-dependencies/**'
+            - '**/*.gradle'
+            - 'mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/**'
+            - 'mobile/android/config/mozconfigs/common*'
+        toolchain-artifact: public/build/android-gradle-dependencies.tar.xz
+        toolchain-alias: android-gradle-dependencies
+    toolchains:
+        # Aliases aren't allowed for toolchains depending on toolchains.
+        - linux64-android-sdk-linux-repack
+        - linux64-proguard-jar-repack
+
 linux64-rust-1.19:
     description: "rust repack"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(rust)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
rename from taskcluster/scripts/builder/build-android-dependencies.sh
rename to taskcluster/scripts/misc/android-gradle-dependencies.sh
--- a/taskcluster/scripts/builder/build-android-dependencies.sh
+++ b/taskcluster/scripts/misc/android-gradle-dependencies.sh
@@ -3,13 +3,24 @@
 set -x -e
 
 echo "running as" $(id)
 
 : WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
 
 set -v
 
-. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/before.sh
+cd $WORKSPACE/build/src
+
+# Download toolchain artifacts.
+. taskcluster/scripts/misc/tooltool-download.sh
 
-. $WORKSPACE/build/src/taskcluster/scripts/builder/build-linux.sh
+# We need a full checkout, and arranging a new sparse profile appears
+# fraught with cache errors.
+hg debugsparse --reset
+
+. taskcluster/scripts/misc/android-gradle-dependencies/before.sh
 
-. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/after.sh
+export MOZCONFIG=mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
+./mach build
+./mach android gradle-dependencies
+
+. taskcluster/scripts/misc/android-gradle-dependencies/after.sh
rename from taskcluster/scripts/builder/build-android-dependencies/after.sh
rename to taskcluster/scripts/misc/android-gradle-dependencies/after.sh
--- a/taskcluster/scripts/builder/build-android-dependencies/after.sh
+++ b/taskcluster/scripts/misc/android-gradle-dependencies/after.sh
@@ -6,30 +6,25 @@ echo "running as" $(id)
 
 : WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
 : GRADLE_VERSION ${GRADLE_VERSION:=2.14.1}
 
 set -v
 
 # Package everything up.
 pushd $WORKSPACE
+mkdir -p android-gradle-dependencies /builds/worker/artifacts
 
-cp -R ${NEXUS_WORK}/storage/jcenter jcenter
-tar cJf jcenter.tar.xz jcenter
-
-cp -R ${NEXUS_WORK}/storage/google google
-tar cJf google.tar.xz google
+cp -R ${NEXUS_WORK}/storage/jcenter android-gradle-dependencies
+cp -R ${NEXUS_WORK}/storage/google android-gradle-dependencies
 
 # The Gradle wrapper will have downloaded and verified the hash of exactly one
 # Gradle distribution.  It will be located in $GRADLE_USER_HOME, like
 # ~/.gradle/wrapper/dists/gradle-2.7-all/$PROJECT_HASH/gradle-2.7-all.zip.  We
 # want to remove the version from the internal directory for use via tooltool in
 # a mozconfig.
-cp $GRADLE_USER_HOME/wrapper/dists/gradle-${GRADLE_VERSION}-all/*/gradle-${GRADLE_VERSION}-all.zip gradle-${GRADLE_VERSION}-all.zip
+cp ${GRADLE_USER_HOME}/wrapper/dists/gradle-${GRADLE_VERSION}-all/*/gradle-${GRADLE_VERSION}-all.zip gradle-${GRADLE_VERSION}-all.zip
 unzip -q gradle-${GRADLE_VERSION}-all.zip
-mv gradle-${GRADLE_VERSION} gradle-dist
-tar cJf gradle-dist.tar.xz gradle-dist
+mv gradle-${GRADLE_VERSION} android-gradle-dependencies/gradle-dist
 
-mkdir -p /builds/worker/artifacts
-mv jcenter.tar.xz /builds/worker/artifacts
-mv google.tar.xz /builds/worker/artifacts
-mv gradle-dist.tar.xz /builds/worker/artifacts
+tar cf - android-gradle-dependencies | xz > /builds/worker/artifacts/android-gradle-dependencies.tar.xz
+
 popd
rename from taskcluster/scripts/builder/build-android-dependencies/before.sh
rename to taskcluster/scripts/misc/android-gradle-dependencies/before.sh
--- a/taskcluster/scripts/builder/build-android-dependencies/before.sh
+++ b/taskcluster/scripts/misc/android-gradle-dependencies/before.sh
@@ -4,17 +4,17 @@ set -x -e
 
 echo "running as" $(id)
 
 : WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
 
 set -v
 
 mkdir -p ${NEXUS_WORK}/conf
-cp /builds/worker/workspace/build/src/taskcluster/scripts/builder/build-android-dependencies/nexus.xml ${NEXUS_WORK}/conf/nexus.xml
+cp /builds/worker/workspace/build/src/taskcluster/scripts/misc/android-gradle-dependencies/nexus.xml ${NEXUS_WORK}/conf/nexus.xml
 
 RUN_AS_USER=worker /opt/sonatype/nexus/bin/nexus restart
 
 # Wait "a while" for Nexus to actually start.  Don't fail if this fails.
 wget --quiet --retry-connrefused --waitretry=2 --tries=100 \
   http://localhost:8081/nexus/service/local/status || true
 rm -rf status
 
rename from taskcluster/scripts/builder/build-android-dependencies/nexus.xml
rename to taskcluster/scripts/misc/android-gradle-dependencies/nexus.xml