Bug 1255227 - Part 2: Bootstrap and upload android-sdk-linux.tar.xz. r=dustin,sebastian
authorNick Alexander <nalexander@mozilla.com>
Wed, 19 Jul 2017 15:45:31 -0700
changeset 422865 23330f35f95778ef167c583ddd3e2c99e96198c2
parent 422864 5b150868209b49cb505d11a5ec8cacc67f90542e
child 422866 f497b6194e9fe4b0e7a20394ef5e1e281a1bac92
push id1517
push userjlorenzo@mozilla.com
push dateThu, 14 Sep 2017 16:50:54 +0000
treeherdermozilla-release@3b41fd564418 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin, sebastian
bugs1255227
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1255227 - Part 2: Bootstrap and upload android-sdk-linux.tar.xz. r=dustin,sebastian Before this patch, we used the Gradle sdk-manager-plugin to download and install Android SDKs and other dependencies. This plugin is now deprecated; the main dependency downloading functionality has been incorporated into the Android-Gradle build plugin. Unfortunately, it's been incorporated into newer versions that in turn require newer toolchains than we currently support, so we can't use the new functionality immediately. Rather than replace sdk-manager-plugin with equivalent Gradle-based functionality, this ticket uses recently added bootstrap functionality to bootstrap the Android SDK during the dependencies task. It then _uses_ that SDK to run the dependency fetching task, _produces_ an android-sdk-linux.tar.xz, and then _uploads_ the new artifact as a private artifact, ready to be pushed into tooltool. This avoids engineers building this critical part of the toolchain locally themselves, and will also feed into ongoing work to push toolchain artifacts into build jobs in Task Cluster. MozReview-Commit-ID: B6FC0ugaCef
mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest
taskcluster/ci/android-stuff/kind.yml
taskcluster/docker/android-gradle-build/bin/after-checkout.sh
taskcluster/docker/android-gradle-build/bin/after.sh
taskcluster/docker/android-gradle-build/bin/build.sh
--- a/mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
+++ b/mobile/android/config/mozconfigs/android-api-15-gradle-dependencies/nightly
@@ -39,8 +39,14 @@ ac_add_options --with-branding=mobile/an
 # This will overwrite the default of stripping everything and keep the symbol table.
 # This is useful for profiling with eideticker. See bug 788680
 STRIP_FLAGS="--strip-debug"
 
 export MOZILLA_OFFICIAL=1
 export MOZ_TELEMETRY_REPORTING=1
 
 . "$topsrcdir/mobile/android/config/mozconfigs/common.override"
+
+# End ../android-api-15-frontend/nightly.
+
+# Populated after checking out the sources and before building the
+# tree as part of the dependencies task bin/ scripts.
+ac_add_options --with-android-sdk="/home/worker/.mozbuild/android-sdk-linux"
--- a/mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest
@@ -1,25 +1,10 @@
 [
   {
-    "versions": [
-      "Android SDK 6.0 / API 23",
-      "Android tools r24.4",
-      "Android build tools 23.0.3",
-      "Android Support Repository (Support Library 23.0.1)",
-      "Google Support Repository (Google Play Services 8.1.0)"
-    ],
-    "size": 573952124,
-    "visibility": "internal",
-    "digest": "1d495d7a7386af3f27b14982e0ff7b0963fd1a63a08040b9b1db0e94c9681fa3704c195ba8be23b5f73e15101b2b767293bc8f96e0584e17867ef13b074e5038",
-    "algorithm": "sha512",
-    "filename": "android-sdk-linux.tar.xz",
-    "unpack": true
-  },
-  {
     "version": "rustc 1.19.0 (0ade33941 2017-07-17) repack",
     "size": 151322128,
     "digest": "1d6d6e8bbf7535a4f1c8f1292562e30c868bfd3ac323992fc287be09f7fa3167c2f9e01dc98143967983e30488fe3ecaebff530ba4ec9113033d60af936c4f4d",
     "algorithm": "sha512",
     "filename": "rustc.tar.xz",
     "unpack": true
   },
   {
--- a/taskcluster/ci/android-stuff/kind.yml
+++ b/taskcluster/ci/android-stuff/kind.yml
@@ -44,16 +44,19 @@ jobs:
                     platform_supports_post_upload_to_latest.py
                 MOZHARNESS_SCRIPT: "mozharness/scripts/fx_desktop_build.py"
                 TOOLTOOL_CACHE: "/home/worker/tooltool-cache"
                 TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest"
             artifacts:
               - name: public/build
                 path: /home/worker/artifacts/
                 type: directory
+              - name: private/android-sdk
+                path: /home/worker/private/android-sdk
+                type: directory
             caches:
               - name: tooltool-cache
                 mount-point: /home/worker/tooltool-cache
                 type: persistent
             relengapi-proxy: true
             command:
               - "/bin/bash"
               - "-c"
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/android-gradle-build/bin/after-checkout.sh
@@ -0,0 +1,10 @@
+#!/bin/bash -vex
+
+set -x -e
+
+: WORKSPACE ${WORKSPACE:=/workspace}
+
+set -v
+
+# Populate /home/worker/.mozbuild/android-sdk-linux.
+python2.7 $WORKSPACE/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive
--- a/taskcluster/docker/android-gradle-build/bin/after.sh
+++ b/taskcluster/docker/android-gradle-build/bin/after.sh
@@ -4,37 +4,38 @@ set -x -e
 
 : WORKSPACE ${WORKSPACE:=/workspace}
 : GRADLE_VERSION ${GRADLE_VERSION:=2.14.1}
 
 set -v
 
 # Package everything up.
 pushd ${WORKSPACE}
-# Not yet.  See notes on tooltool below.
-# cp -R /root/.android-sdk android-sdk-linux
-# tar cJf android-sdk-linux.tar.xz android-sdk-linux
+
+cp -R /home/worker/.mozbuild/android-sdk-linux android-sdk-linux
+tar cJf android-sdk-linux.tar.xz android-sdk-linux
+
+# We can't redistribute the Android SDK publicly.
+mkdir -p /home/worker/private/android-sdk
+mv android-sdk-linux.tar.xz /home/worker/private/android-sdk
 
 cp -R /workspace/nexus/storage/jcenter jcentral
 tar cJf jcentral.tar.xz jcentral
 
 # The Gradle wrapper will have downloaded and verified the hash of exactly one
 # Gradle distribution.  It will be located in $GRADLE_USER_HOME, like
 # ~/.gradle/wrapper/dists/gradle-2.7-all/$PROJECT_HASH/gradle-2.7-all.zip.  We
 # want to remove the version from the internal directory for use via tooltool in
 # a mozconfig.
 cp $GRADLE_USER_HOME/wrapper/dists/gradle-${GRADLE_VERSION}-all/*/gradle-${GRADLE_VERSION}-all.zip gradle-${GRADLE_VERSION}-all.zip
 unzip -q gradle-${GRADLE_VERSION}-all.zip
 mv gradle-${GRADLE_VERSION} gradle-dist
 tar cJf gradle-dist.tar.xz gradle-dist
 
 mkdir -p /home/worker/artifacts
-# We can't redistribute the Android SDK publicly just yet.  We'll
-# upload to (internal) tooltool eventually.  mv
-# android-sdk-linux.tar.xz /home/worker/artifacts
 mv jcentral.tar.xz /home/worker/artifacts
 mv gradle-dist.tar.xz /home/worker/artifacts
 popd
 
 # Bug 1245170: at some point in the future, we'll be able to upload
 # things directly to tooltool.
 # pushd /home/worker/artifacts
 # /build/tooltool.py add --visibility=public jcentral.tar.xz
--- a/taskcluster/docker/android-gradle-build/bin/build.sh
+++ b/taskcluster/docker/android-gradle-build/bin/build.sh
@@ -21,9 +21,11 @@ fi
 
 ####
 # The default build works for any fx_desktop_build based mozharness job:
 # via linux-build.sh
 ####
 
 . $HOME/bin/checkout-sources.sh
 
+. $HOME/bin/after-checkout.sh
+
 . $WORKSPACE/build/src/taskcluster/scripts/builder/build-linux.sh