Bug 1508976 - Produce a multi-architecture GeckoView "fat AAR". r=snorp,agi,froydnj
☠☠ backed out by be4b780a7783 ☠ ☠
authorNick Alexander <nalexander@mozilla.com>
Tue, 05 Mar 2019 17:52:54 +0000
changeset 462412 4e5d97c935151d28d1e06943133951918ac28568
parent 462411 1a681dc60b60a4040959de4b44fa8d6a4f8f08e1
child 462413 da57df805c568ba2af98d1d62fc3756584e84682
push id112307
push usershindli@mozilla.com
push dateWed, 06 Mar 2019 05:00:25 +0000
treeherdermozilla-inbound@daf32259f33e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssnorp, agi, froydnj
bugs1508976
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1508976 - Produce a multi-architecture GeckoView "fat AAR". r=snorp,agi,froydnj Differential Revision: https://phabricator.services.mozilla.com/D15771
build.gradle
mobile/android/geckoview/build.gradle
mobile/android/geckoview/fat_aar.gradle
mobile/android/moz.configure
taskcluster/ci/build/android.yml
--- a/build.gradle
+++ b/build.gradle
@@ -81,16 +81,18 @@ buildscript {
     }
 
     dependencies {
         classpath 'org.mozilla.apilint:apilint:0.1.7'
         classpath 'com.android.tools.build:gradle:3.1.4'
         classpath 'com.getkeepsafe.dexcount:dexcount-gradle-plugin:0.8.2'
         classpath 'org.apache.commons:commons-exec:1.3'
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+        // Downloading archives from Taskcluster.
+        classpath 'de.undercouch:gradle-download-task:3.4.3'
     }
 }
 
 // A stream that processes bytes line by line, prepending a tag before sending
 // each line to Gradle's logging.
 class TaggedLogOutputStream extends org.apache.commons.exec.LogOutputStream {
     String tag
     Logger logger
--- a/mobile/android/geckoview/build.gradle
+++ b/mobile/android/geckoview/build.gradle
@@ -1,10 +1,26 @@
 buildDir "${topobjdir}/gradle/build/mobile/android/geckoview"
 
+buildscript {
+    repositories {
+        gradle.mozconfig.substs.GRADLE_MAVEN_REPOSITORIES.each { repository ->
+            maven {
+                url repository
+            }
+        }
+    }
+
+    // Used in fat_aar.gradle.  See
+    // http://gradle.1045684.n5.nabble.com/Issue-with-imports-in-external-build-scripts-tp3322487p3322906.html.
+    dependencies {
+        classpath 'commons-codec:commons-codec:1.11'
+    }
+}
+
 apply plugin: 'com.android.library'
 apply plugin: 'kotlin-android'
 
 apply from: "${topsrcdir}/mobile/android/gradle/product_flavors.gradle"
 
 // The SDK binding generation tasks depend on the JAR creation task of the
 // :annotations project.
 evaluationDependsOn(':annotations')
@@ -512,8 +528,20 @@ apply plugin: 'org.mozilla.apilint'
 
 apiLint {
     // TODO: Change this to `org` after hiding org.mozilla.gecko
     packageFilter = 'org.mozilla.geckoview'
     changelogFileName = 'src/main/java/org/mozilla/geckoview/doc-files/CHANGELOG.md'
     skipClassesRegex = ['^org.mozilla.geckoview.BuildConfig$']
     lintFilters = ['GV']
 }
+
+// Plugin is always applied so that its dependencies are downloaded, even if the
+// functionality isn't used.
+apply plugin: 'de.undercouch.download'
+
+if (mozconfig.substs.MOZ_ANDROID_FAT_AAR_TARGETS) {
+    // This is frustrating, but here we are: frustrated.  See
+    // https://stackoverflow.com/a/37060550.
+    ext.DigestUtils = org.apache.commons.codec.digest.DigestUtils
+
+    apply from: "${topsrcdir}/mobile/android/geckoview/fat_aar.gradle"
+}
new file mode 100644
--- /dev/null
+++ b/mobile/android/geckoview/fat_aar.gradle
@@ -0,0 +1,207 @@
+// Produce a multi-architecture universal "fat AAR".
+//
+// This uses Gradle to:
+//
+// 1) download per-architecture AARs
+// 2) ensure that they are "compatible"
+// 3) pack the per-architecture JNI libraries into the GeckoView library
+
+import org.gradle.api.file.FileTree
+
+Set contentHashes(FileTree tree) {
+    def Set hashes = []
+    tree.visit {
+        if (!it.isDirectory()) {
+            hashes << [it.relativePath.pathString, ext.DigestUtils.md5Hex(it.open())]
+        }
+    }
+    return hashes
+}
+
+Set contentDiff(FileTree left, FileTree right) {
+    def Set hleft = contentHashes(left)
+    def Set hright = contentHashes(right)
+    return (hleft + hright) - hleft.intersect(hright)
+}
+
+def ANDROID_CPU_ARCH = mozconfig.substs.ANDROID_CPU_ARCH
+def MOZ_ANDROID_FAT_AAR_TARGETS = mozconfig.substs.MOZ_ANDROID_FAT_AAR_TARGETS
+
+task downloadMavenZips
+task unpackMavenZips
+
+// Turn ["arch1=url1", "arch2=url2"] into ["arch1": "url1", "arch2": "url2"].
+def srcs = MOZ_ANDROID_FAT_AAR_TARGETS.collectEntries {
+    def pair = it.split('=')
+    [(pair.first()): pair.last()]
+}
+
+if (!srcs.containsKey(ANDROID_CPU_ARCH)) {
+    throw new GradleException("MOZ_ANDROID_FAT_AAR_TARGETS set but current architecture (${ANDROID_CPU_ARCH}) not included: '${MOZ_ANDROID_FAT_AAR_TARGETS}'")
+}
+
+for (s in srcs) {
+    def downloadMavenZipTask = task "downloadMavenZip${s.key.capitalize()}"(type: Download) {
+        src s.value
+        dest file("${buildDir}/fat_aar/${s.key}-${s.value.hashCode()}.maven.zip")
+
+        doFirst {
+            if (it.dest.exists()) {
+                throw new StopExecutionException("File to download already exists: ${it.dest.path}")
+            }
+        }
+        overwrite true
+    }
+    downloadMavenZips.dependsOn(downloadMavenZipTask)
+
+    // This is awkward, but extracting from nested zipTree instances seems to be
+    // very challenging: Gradle really, really, really wants to resolve the
+    // internal zipTree dependencies at configure time, before the outer zipTree
+    // has been downloaded.
+    def unpackMavenZipTask = task "unpackMavenZip${s.key.capitalize()}"(dependsOn: downloadMavenZipTask) {
+        inputs.files downloadMavenZipTask.dest
+        outputs.dir file("${buildDir}/fat_aar/${s.key}")
+
+        // Capture for the closure below.
+        def key = s.key
+
+        // sync { ... } from zipTree doesn't seem to be supported.
+        doFirst {
+            delete file("${buildDir}/fat_aar/${key}")
+        }
+
+        doLast {
+            copy {
+                from zipTree(zipTree(tasks["downloadMavenZip${key.capitalize()}"].dest).files.find { it.name.endsWith('.aar') })
+                into file("${buildDir}/fat_aar/${key}")
+            }
+        }
+    }
+    unpackMavenZips.dependsOn(unpackMavenZipTask)
+}
+
+task checkAARs(dependsOn: unpackMavenZips) {
+    inputs.files unpackMavenZips.outputs.files
+
+    doFirst {
+        def base = fileTree("${buildDir}/fat_aar/${ANDROID_CPU_ARCH}")
+
+        for (s in srcs) {
+            if (s.key == ANDROID_CPU_ARCH) {
+                continue
+            }
+
+            def other = fileTree("${buildDir}/fat_aar/${s.key}")
+
+            // So:
+            // - obviously the JNI libraries themselves are different across architectures
+            // - AndroidManifest.xml embeds MIN_SDK_VERSION, which differs on 64-bit
+            //   architectures (21) vs non-64-bit architectures (15)
+            // - classes.jar also embeds MIN_SDK_VERSION, which differs on 64-bit
+            //   architectures (21) vs non-64-bit architectures (15)
+            // - annotations.zip doesn't have constant timestamps
+            // - assets/omni.ja is actually different (buildconfig.html embeds target architecture)
+            //
+            // For reasons unknown, AARs have an empty top-level $ARCH
+            // directory, but we ignore directories so it needs no special
+            // accommodation.
+            def diff = contentDiff(
+                base
+                    .exclude('jni/**')
+                    .exclude('AndroidManifest.xml')
+                    .exclude('classes.jar')
+                    .exclude('annotations.zip')
+                    .exclude('assets/omni.ja'),
+                other
+                    .exclude('jni/**')
+                    .exclude('AndroidManifest.xml')
+                    .exclude('classes.jar')
+                    .exclude('annotations.zip')
+                    .exclude('assets/omni.ja'))
+
+            if (!diff.isEmpty()) {
+                throw new GradleException("${ANDROID_CPU_ARCH} AAR and ${s.key} AAR disagree: ${diff}")
+            }
+
+            def classes = contentDiff(
+                zipTree("${buildDir}/fat_aar/${ANDROID_CPU_ARCH}/classes.jar")
+                    .matching {
+                        exclude('org/mozilla/geckoview/BuildConfig.class')
+                        exclude('org/mozilla/gecko/util/HardwareUtils.class')
+                    },
+                zipTree("${buildDir}/fat_aar/${s.key}/classes.jar")
+                    .matching {
+                        exclude('org/mozilla/geckoview/BuildConfig.class')
+                        exclude('org/mozilla/gecko/util/HardwareUtils.class')
+                    })
+
+            if (!classes.isEmpty()) {
+                throw new GradleException("${buildDir}/fat_aar/${ANDROID_CPU_ARCH}/classes.jar and ${buildDir}/fat_aar/${s.key}/classes.jar disagree: ${classes}")
+            }
+
+            def annotations = contentDiff(
+                zipTree("${buildDir}/fat_aar/${ANDROID_CPU_ARCH}/annotations.zip"),
+                zipTree("${buildDir}/fat_aar/${s.key}/annotations.zip"))
+
+            if (!annotations.isEmpty()) {
+                throw new GradleException("${buildDir}/fat_aar/${ANDROID_CPU_ARCH}/annotations.zip and ${buildDir}/fat_aar/${s.key}/annotations.zip disagree: ${annotations}")
+            }
+
+            // buildconfig.html is special.
+            def omnijar = contentDiff(
+                zipTree("${buildDir}/fat_aar/${ANDROID_CPU_ARCH}/assets/omni.ja")
+                    .matching {
+                        exclude('**/buildconfig.html')
+                    },
+                zipTree("${buildDir}/fat_aar/${s.key}/assets/omni.ja")
+                    .matching {
+                        exclude('**/buildconfig.html')
+                    })
+
+            if (!omnijar.isEmpty()) {
+                // Let's make this easy to debug locally, at least.
+                def relPaths = omnijar.collectEntries().keySet()
+
+                copy {
+                    from zipTree("${buildDir}/fat_aar/${ANDROID_CPU_ARCH}/assets/omni.ja")
+                    into file("${buildDir}/fat_aar/${ANDROID_CPU_ARCH}/omnijar")
+                    for (relPath in relPaths) {
+                        include relPath
+                    }
+                }
+
+                copy {
+                    from zipTree("${buildDir}/fat_aar/${s.key}/assets/omni.ja")
+                    into file("${buildDir}/fat_aar/${s.key}/omnijar")
+                    for (relPath in relPaths) {
+                        include relPath
+                    }
+                }
+
+                def pairs = relPaths.collect { relPath ->
+                    "(${buildDir}/fat_aar/${ANDROID_CPU_ARCH}/omnijar/${relPath}, ${buildDir}/fat_aar/${s.key}/omnijar/${relPath})"
+                }
+
+                throw new GradleException("${buildDir}/fat_aar/${ANDROID_CPU_ARCH}/assets/omni.ja and ${buildDir}/fat_aar/${s.key}/assets/omni.ja disagree: ${pairs}")
+            }
+        }
+    }
+}
+
+android.libraryVariants.all { variant ->
+    if ((variant.productFlavors*.name).contains('withGeckoBinaries')) {
+        def assetGenTask = tasks.findByName("generate${variant.name.capitalize()}Assets")
+        assetGenTask.dependsOn checkAARs
+
+        for (s in srcs) {
+            // We want to ensure that nothing changed between this build and
+            // the upstream AAR, but don't want to pack the binaries
+            // corresponding to this build's architecture twice.
+            if (s.key == ANDROID_CPU_ARCH) {
+                continue
+            }
+
+            android.sourceSets."${variant.name}".jniLibs.srcDir "${buildDir}/fat_aar/${s.key}/jni"
+        }
+    }
+}
--- a/mobile/android/moz.configure
+++ b/mobile/android/moz.configure
@@ -169,8 +169,23 @@ def check_android_pocket(android_pocket,
 # Must come after the ../../toolkit/moz.configure.
 @depends('MOZ_ANDROID_GCM', google_play_services)
 def check_android_gcm(android_gcm,
                       google_play_services):
     if android_gcm:
         if not google_play_services:
             die('You must specify --with-google-play-services when'
                 ' building with MOZ_ANDROID_GCM=1')
+
+# Automation will set this to ("arch=https://...", ...) via the TC environment.
+option(env='MOZ_ANDROID_FAT_AAR_TARGETS',
+       nargs='*',
+       help='Comma-separated arch=URL pairs mapping Android architectures '
+            'to URLs to maven.target.zip archives.')
+
+@depends('MOZ_ANDROID_FAT_AAR_TARGETS')
+def fat_aar_targets(values):
+    for value in values:
+        if not value.count('=') == 1:
+            die('MOZ_ANDROID_FAT_AAR_TARGETS entries must be of the form "arch=URL": "{}"'.format(value))
+    return values
+
+set_config('MOZ_ANDROID_FAT_AAR_TARGETS', fat_aar_targets)
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -777,8 +777,72 @@ android-x86_64/debug:
         - android-sdk-linux
         - linux64-clang
         - linux64-rust-android
         - linux64-rust-size
         - linux64-cbindgen
         - linux64-sccache
         - linux64-nasm
         - linux64-node
+
+android-geckoview-fat-aar/opt:
+    description: "Android GeckoView multi-architecture fat AAR Opt"
+    index:
+        product: mobile
+        job-name: android-geckoview-fat-aar-opt
+    treeherder:
+        platform: android-4-0-geckoview-fat-aar/opt
+        symbol: Bgv
+    worker-type: aws-provisioner-v1/gecko-{level}-b-android
+    dependencies:
+        android-x86: build-android-x86/opt
+        android-x86_64: build-android-x86_64/opt
+        android-api-16: build-android-api-16/opt
+        android-aarch64: build-android-aarch64/opt
+    worker:
+        docker-image: {in-tree: android-build}
+        max-run-time: 7200
+        env:
+            # Online in order to download the per-architecture AARs.
+            GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online"
+            TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest"
+            PERFHERDER_EXTRA_OPTIONS: android-geckoview-fat-aar-opt
+            MOZ_ANDROID_FAT_AAR_TARGETS: {artifact-reference: "x86=<android-x86/public/build/target.maven.zip>,\
+                                                               x86_64=<android-x86_64/public/build/target.maven.zip>,\
+                                                               armeabi-v7a=<android-api-16/public/build/target.maven.zip>,\
+                                                               arm64-v8a=<android-aarch64/public/build/target.maven.zip>"}
+        artifacts:
+            - name: public/android/maven
+              # TODO Bug 1433198. Remove the following entry once target.maven.zip is uploaded to a maven repository
+              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/maven/
+              type: directory
+            - name: public/build/target.maven.zip
+              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/target.maven.zip
+              type: file
+            - name: public/build/geckoview-androidTest.apk
+              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk
+              type: file
+            - name: public/build/geckoview_example.apk
+              path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk
+              type: file
+            - name: public/build
+              path: /builds/worker/artifacts/
+              type: directory
+    run:
+        using: mozharness
+        actions: [get-secrets, build, multi-l10n]
+        config:
+            - builds/releng_base_android_64_builds.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        custom-build-variant-cfg: api-16
+        tooltool-downloads: internal
+    toolchains:
+        - android-gradle-dependencies
+        - android-ndk-linux
+        - android-sdk-linux
+        - linux64-clang
+        - linux64-rust-android
+        - linux64-rust-size
+        - linux64-cbindgen
+        - linux64-sccache
+        - linux64-nasm
+        - linux64-node