Merge mozilla-central to mozilla-inbound. a=merge on a CLOSED TREE
authorRazvan Maries <rmaries@mozilla.com>
Fri, 30 Nov 2018 05:15:27 +0200
changeset 448939 2535c232ed6e9109ae790d3b795b6f746505d5c8
parent 448938 b26e28ce8cfebb91f1c5764606585b8b670938ff (current diff)
parent 448848 e18555e129e755fd2fa06e2760f05d4ebafe4187 (diff)
child 448940 6baf9f33a6cd6f5a170c1bc6fcd2d67019d598af
push id35130
push usernerli@mozilla.com
push dateFri, 30 Nov 2018 09:35:34 +0000
treeherdermozilla-central@58a0412e1557 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmerge
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Merge mozilla-central to mozilla-inbound. a=merge on a CLOSED TREE
--- a/build.gradle
+++ b/build.gradle
@@ -53,132 +53,54 @@ buildscript {
     if (gradle.mozconfig.substs.MOZ_ANDROID_GOOGLE_PLAY_SERVICES) {
         ext.google_play_services_version = '15.0.1'
     }
 
     dependencies {
         classpath 'org.mozilla.apilint:apilint:0.1.4'
         classpath 'com.android.tools.build:gradle:3.1.4'
         classpath 'com.getkeepsafe.dexcount:dexcount-gradle-plugin:0.8.2'
-        classpath 'org.apache.commons:commons-exec:1.3'
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
     }
 }
 
-// A stream that processes bytes line by line, prepending a tag before sending
-// each line to Gradle's logging.
-class TaggedLogOutputStream extends org.apache.commons.exec.LogOutputStream {
-    String tag
-    Logger logger
-
-    TaggedLogOutputStream(tag, logger) {
-        this.tag = tag
-        this.logger = logger
-    }
-
-    void processLine(String line, int level) {
-        logger.lifecycle("${this.tag} ${line}")
-    }
-}
-
-ext.geckoBinariesOnlyIf = { task ->
-    // Never for official builds.
-    if (mozconfig.substs.MOZILLA_OFFICIAL) {
-        rootProject.logger.lifecycle("Skipping task ${task.path} because: MOZILLA_OFFICIAL")
-        return false
-    }
-
+if ('multi' == System.env.AB_CD) {
     // Multi-l10n builds set `AB_CD=multi`, which isn't a valid locale.  This
     // causes the
     //
-    // |mach build| > |mach gradle| > AndroidManifest.xml > strings.xml > multi/brand.dtd
+    // |mach build| > |mach gradle| > |make gradle-targets| > AndroidManifest.xml > strings.xml > multi/brand.dtd
     //
     // dependency chain to fail, since multi isn't a real locale.  To avoid
     // this, if Gradle is invoked with AB_CD=multi, we don't invoke Make at all.
-    if ('multi' == System.env.AB_CD) {
-        rootProject.logger.lifecycle("Skipping task ${task.path} because: AB_CD=multi")
-        return false
-    }
-
+    task generateCodeAndResources()
+} else if (System.env.IS_LANGUAGE_REPACK == '1') {
     // Single-locale l10n repacks set `IS_LANGUAGE_REPACK=1` and handle resource
     // and code generation themselves.
-    if ('1' == System.env.IS_LANGUAGE_REPACK) {
-        rootProject.logger.lifecycle("Skipping task ${task.path} because: IS_LANGUAGE_REPACK")
-        return false
-    }
-
-    rootProject.logger.lifecycle("Executing task ${task.path}")
-    return true
-}
-
-task machBuildFaster(type: Exec) {
-    onlyIf rootProject.ext.geckoBinariesOnlyIf
-
-    workingDir "${topsrcdir}"
-
-    commandLine mozconfig.substs.PYTHON
-    args "${topsrcdir}/mach"
-    args 'build'
-    args 'faster'
-
-    // Add `-v` if we're running under `--info` (or `--debug`).
-    if (project.logger.isEnabled(LogLevel.INFO)) {
-        args '-v'
-    }
-
-    // `path` is like `:machBuildFaster`.
-    standardOutput = new TaggedLogOutputStream("${path}>", logger)
-    errorOutput = standardOutput
-}
-
-def createMachStagePackageTask(name) {
-    return task(name, type: Exec) {
-        onlyIf rootProject.ext.geckoBinariesOnlyIf
-
-        dependsOn rootProject.machBuildFaster
-
-        // We'd prefer to take these from the :omnijar project directly, but
-        // it's awkward to reach across projects at evaluation time, so we
-        // duplicate the list here.
-        inputs.dir "${topsrcdir}/mobile/android/chrome"
-        inputs.dir "${topsrcdir}/mobile/android/components"
-        inputs.dir "${topsrcdir}/mobile/android/locales"
-        inputs.dir "${topsrcdir}/mobile/android/modules"
-        inputs.dir "${topsrcdir}/mobile/android/themes"
-        inputs.dir "${topsrcdir}/toolkit"
-
+    task generateCodeAndResources()
+} else {
+    task generateCodeAndResources(type:Exec) {
         workingDir "${topobjdir}"
 
-        // We'd prefer this to be a `mach` invocation, but `mach build
-        // mobile/android/installer/stage-package` doesn't work as expected.
         commandLine mozconfig.substs.GMAKE
         args '-C'
-        args "${topobjdir}/mobile/android/installer"
-        args 'stage-package'
+        args "${topobjdir}/mobile/android/base"
+        args 'gradle-targets'
 
-        outputs.file "${topobjdir}/dist/fennec/assets/${mozconfig.substs.ANDROID_CPU_ARCH}/libxul.so"
-        outputs.file "${topobjdir}/dist/fennec/lib/${mozconfig.substs.ANDROID_CPU_ARCH}/libmozglue.so"
-
-        // `path` is like `:machStagePackage`.
-        standardOutput = new TaggedLogOutputStream("${path}>", logger)
+        // Only show the output if something went wrong.
+        ignoreExitValue = true
+        standardOutput = new ByteArrayOutputStream()
         errorOutput = standardOutput
+        doLast {
+            if (execResult.exitValue != 0) {
+                throw new GradleException("Process '${commandLine}' finished with non-zero exit value ${execResult.exitValue}:\n\n${standardOutput.toString()}")
+            }
+        }
     }
 }
 
-createMachStagePackageTask("machStagePackageForFennec").with {
-    outputs.file "${topobjdir}/dist/fennec/assets/omni.ja"
-}
-
-createMachStagePackageTask("machStagePackageForGeckoview").with {
-    args 'MOZ_GECKOVIEW_JAR=1'
-    outputs.file "${topobjdir}/dist/geckoview/assets/omni.ja"
-    // Avoid races between stage-package invocations.
-    mustRunAfter tasks["machStagePackageForFennec"]
-}
-
 afterEvaluate {
     subprojects { project ->
         if (project.name != 'thirdparty') {
             tasks.withType(JavaCompile) {
                 // Add compiler args for all code except third-party code.
                 options.compilerArgs += [
                     // Turn on all warnings, except...
                     "-Xlint:all",
@@ -204,20 +126,20 @@ afterEvaluate {
                 }
             }
         }
 
         if (!hasProperty('android')) {
             return
         }
         android.applicationVariants.all {
-            preBuild.dependsOn rootProject.machBuildFaster
+            preBuild.dependsOn rootProject.generateCodeAndResources
         }
         android.libraryVariants.all {
-            preBuild.dependsOn rootProject.machBuildFaster
+            preBuild.dependsOn rootProject.generateCodeAndResources
         }
     }
 }
 
 apply plugin: 'idea'
 
 idea {
     project {
--- a/config/faster/rules.mk
+++ b/config/faster/rules.mk
@@ -61,23 +61,16 @@ endif
 # Extra define to trigger some workarounds. We should strive to limit the
 # use of those. As of writing the only one is in browser/locales/jar.mn.
 ACDEFINES += -DBUILD_FASTER
 
 # Files under the faster/ sub-directory, however, are not meant to use the
 # fallback
 $(TOPOBJDIR)/faster/%: ;
 
-ifeq ($(MOZ_BUILD_APP),mobile/android)
-# The generic rule doesn't handle relative directories, which are used
-# extensively in mobile/android/base.
-$(TOPOBJDIR)/mobile/android/base/% : $(TOPOBJDIR)/buildid.h FORCE
-	$(MAKE) -C $(TOPOBJDIR)/mobile/android/base $*
-endif
-
 # Generic rule to fall back to the recursive make backend.
 # This needs to stay after other $(TOPOBJDIR)/* rules because GNU Make
 # <3.82 apply pattern rules in definition order, not stem length like
 # modern GNU Make.
 $(TOPOBJDIR)/%: FORCE
 	$(MAKE) -C $(dir $@) $(notdir $@)
 
 # Install files using install manifests
--- a/mobile/android/app/build.gradle
+++ b/mobile/android/app/build.gradle
@@ -333,17 +333,17 @@ android.applicationVariants.all { varian
     // `sourceSets` task, for reasons unknown.
     variant.registerGeneratedResFolders(project.files(syncPreprocessedRes.destinationDir).builtBy(syncPreprocessedRes))
 
     // It's not easy -- see the backout in Bug 1242213 -- to change the
     // <manifest> package for Fennec.  Gradle has grown a mechanism to achieve
     // what we want for Fennec, however, with applicationId.  To use the same
     // manifest as moz.build, we replace the package with org.mozilla.gecko (the
     // eventual package) here.
-    def rewriteManifestPackage = task("rewriteManifestPackageFor${variant.name.capitalize()}", type: Copy, dependsOn: rootProject.machBuildFaster) {
+    def rewriteManifestPackage = task("rewriteManifestPackageFor${variant.name.capitalize()}", type: Copy, dependsOn: rootProject.generateCodeAndResources) {
         into("${project.buildDir}/moz.build/src/${variant.name}")
         from("${topobjdir}/mobile/android/base/AndroidManifest.xml")
         filter { it.replaceFirst(/package=".*?"/, 'package="org.mozilla.gecko"') }
         exclude('**/*.mkdir.done')
     }
 
     // Every configuration needs the stub manifest at
     // src/main/AndroidManifest.xml and the generated manifest.  We can't use
@@ -354,19 +354,19 @@ android.applicationVariants.all { varian
 
 
     // Local (read, not 'official') builds want to reflect developer changes to
     // AndroidManifest.xml.in, strings.xml, and preprocessed Java code.  To do
     // this, the Gradle build calls out to the moz.build system, which can be
     // re-entrant.  Official builds are driven by the moz.build system and
     // should never be re-entrant in this way.
     if (!mozconfig.substs.MOZILLA_OFFICIAL) {
-        syncPreprocessedJava.dependsOn rootProject.machBuildFaster
-        syncPreprocessedRes.dependsOn rootProject.machBuildFaster
-        rewriteManifestPackage.dependsOn rootProject.machBuildFaster
+        syncPreprocessedJava.dependsOn rootProject.generateCodeAndResources
+        syncPreprocessedRes.dependsOn rootProject.generateCodeAndResources
+        rewriteManifestPackage.dependsOn rootProject.generateCodeAndResources
     }
 
     // When driven from moz.build via |mach build|, Gradle does not require or
     // use Gecko binaries.  It's only |mach package| that packs the Gecko
     // binaries into the resulting APK.  The "withoutGeckoBinaries" variants
     // handle this.  When driven from Android Studio or Gradle, the
     // "withGeckoBinaries" variants handle packing the Gecko binaries into the
     // resulting APK (for on-device deployment).  They also update the Omnijars
--- a/mobile/android/app/omnijar/build.gradle
+++ b/mobile/android/app/omnijar/build.gradle
@@ -1,25 +1,24 @@
 buildDir "${topobjdir}/gradle/build/mobile/android/omnijar"
 
 apply plugin: 'java'
 
-// This project is a dummy project; the JAR produced is not used.  The root
+// This project is a dummy project; the JAR produced is not used.  The :app
 // project uses the set of inputs here to check if the omnijar needs to be
 // rebuilt.  By listing them here as resource directories, IntelliJ labels each
 // checked directly nicely.  Why list the directories here?  There's a mismatch
 // between SourceDirectorySet and TaskInputs: the former is directory oriented,
 // while the latter is more general.  That means its easy to convert this list
-// into inputs for the root project, but not vice-versa.  Sadly this would mean
-// that the root project evaluation depends on :omnijar, so we duplicate in the
-// root project.
+// into inputs for :app, but not vice-versa.  Sadly this implies that :app
+// evaluation depends on :omnijar, but the evaluation overhead is low enough
+// that we accept it.
 sourceSets {
     main {
-        // These are supposed to be files that go into the omnijar.  But yes,
-        // toolkit/ has lots of native code.
+        // Depend on the Gecko resources in mobile/android.
         resources {
             srcDir "${topsrcdir}/mobile/android/chrome"
             srcDir "${topsrcdir}/mobile/android/components"
             srcDir "${topsrcdir}/mobile/android/locales"
             srcDir "${topsrcdir}/mobile/android/modules"
             srcDir "${topsrcdir}/mobile/android/themes"
             srcDir "${topsrcdir}/toolkit"
         }
--- a/mobile/android/base/Makefile.in
+++ b/mobile/android/base/Makefile.in
@@ -1,20 +1,65 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # Developer builds call mach -> Make -> gradle -> mach, which races to
 # find and create .mozconfig files and to generate targets.
 .NOTPARALLEL:
 
+generated_resources := \
+  AndroidManifest.xml \
+  res/raw/suggestedsites.json \
+  res/values/strings.xml \
+  $(NULL)
+
+generated_files := \
+  AndroidManifest.xml \
+  generated/preprocessed/org/mozilla/gecko/AdjustConstants.java \
+  generated/preprocessed/org/mozilla/gecko/AppConstants.java \
+  generated/preprocessed/org/mozilla/gecko/MmaConstants.java \
+  $(NULL)
+
 include $(topsrcdir)/config/AB_rCD.mk
 
 chrome-%:: AB_CD=$*
 chrome-%::
 	$(MAKE) \
 	  res/values$(AB_rCD)/strings.xml \
 	  res/raw$(AB_rCD)/suggestedsites.json \
 	  AB_CD=$*
 
 include $(topsrcdir)/config/rules.mk
 
+# This target is only used by IDE integrations. It rebuilds resources
+# that end up in omni.ja using the equivalent of |mach build faster|,
+# does most of the packaging step, and then updates omni.ja in
+# place. If you're not using an IDE, you should be using |mach build
+# mobile/android && mach package|.
+$(ABS_DIST)/fennec/$(OMNIJAR_NAME): FORCE
+	$(REPORT_BUILD)
+	$(MAKE) -C ../../../faster
+	$(MAKE) -C ../installer stage-package
+	$(MKDIR) -p $(@D)
+	rsync --update $(DIST)/fennec/$(notdir $(OMNIJAR_NAME)) $@
+	$(RM) $(DIST)/fennec/$(notdir $(OMNIJAR_NAME))
+
+ifndef MOZILLA_OFFICIAL
+# Targets built very early during a Gradle build.  In automation,
+# these are built before Gradle is invoked, and gradle-targets is not
+# made at all.  This is required to avoid building gradle-targets with
+# AB_CD=multi during multi-l10n builds.
+gradle-targets: $(generated_resources) $(generated_files)
+
+# Local developers update omni.ja during their builds.  There's a
+# chicken-and-egg problem here.
+gradle-omnijar: $(abspath $(DIST)/fennec/$(OMNIJAR_NAME))
+else
+# In automation, omni.ja is built only during packaging.
+gradle-omnijar:
+
+gradle-targets:
+endif
+
+.PHONY: gradle-targets gradle-omnijar
+
 export:: android_apks
--- a/mobile/android/base/moz.build
+++ b/mobile/android/base/moz.build
@@ -178,38 +178,31 @@ for f in ['res/values/strings.xml',
     strings.script = '/python/mozbuild/mozbuild/action/generate_strings_xml.py'
     strings.inputs = [
         'strings.xml.in',
         # The `locales/en-US/` will be rewritten to the locale-specific path.
         'locales/en-US/android_strings.dtd',
         'locales/en-US/sync_strings.dtd',
     ]
 
-generated_inputs = [
+# The recursive make backend treats the first output specially: it's passed as
+# an open FileAvoidWrite to the invoked script.  That doesn't work well with
+# the Gradle task that generates all of the outputs, so we add a dummy first
+# output.
+t = ('android_apks',
+     CONFIG['GRADLE_ANDROID_APP_APK'],
+     CONFIG['GRADLE_ANDROID_APP_ANDROIDTEST_APK'])
+
+GENERATED_FILES += [t]
+GENERATED_FILES[t].force = True
+GENERATED_FILES[t].script = '/mobile/android/gradle.py:assemble_app'
+GENERATED_FILES[t].inputs += [
     '!AndroidManifest.xml',
     '!generated/preprocessed/org/mozilla/gecko/AdjustConstants.java',
     '!generated/preprocessed/org/mozilla/gecko/AppConstants.java',
     '!generated/preprocessed/org/mozilla/gecko/CrashReporterConstants.java',
     '!generated/preprocessed/org/mozilla/gecko/MmaConstants.java',
     # These all depend on AB_CD, which isn't captured in this definition.  Due
     # to subtle RecursiveMake details, everything works out.  In the future we
     # can try to express the APKs themselves as LOCALIZED_GENERATED_FILES.
     '!res/raw/suggestedsites.json',
     '!res/values/strings.xml',
 ]
-
-# The recursive make backend treats the first output specially: it's passed as
-# an open FileAvoidWrite to the invoked script.  That doesn't work well with
-# the Gradle task that generates all of the outputs, so we add a dummy first
-# output.
-t = ('android_apks',
-     CONFIG['GRADLE_ANDROID_APP_APK'],
-     CONFIG['GRADLE_ANDROID_APP_ANDROIDTEST_APK'])
-
-GENERATED_FILES += [t]
-GENERATED_FILES[t].force = True
-GENERATED_FILES[t].script = '/mobile/android/gradle.py:assemble_app'
-GENERATED_FILES[t].inputs += generated_inputs
-
-# This makes |mach build faster| build these files.  See the workaround in
-# config/faster/rules.mk.  None of these files appear in package-manifest.in,
-# so they're not packaged.
-FINAL_TARGET_FILES.android += generated_inputs
--- a/mobile/android/gradle/with_gecko_binaries.gradle
+++ b/mobile/android/gradle/with_gecko_binaries.gradle
@@ -1,16 +1,49 @@
 /* -*- Mode: Groovy; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+// The omnijar inputs are listed as resource directory inputs to a dummy JAR.
+// That arrangement labels them nicely in IntelliJ.  See the comment in the
+// :omnijar project for more context.
+evaluationDependsOn(':omnijar')
+
 // The JNI wrapper generation tasks depend on the JAR creation task of the :annotations project.
 evaluationDependsOn(':annotations')
 
+task buildOmnijars(type:Exec) {
+    dependsOn rootProject.generateCodeAndResources
+
+    // See comment in :omnijar project regarding interface mismatches here.
+    inputs.file(project(':omnijar').sourceSets.main.resources.srcDirs).skipWhenEmpty() 
+
+    // Produce both the Fennec and the GeckoView omnijars.
+    outputs.file "${topobjdir}/dist/fennec/assets/omni.ja"
+    outputs.file "${topobjdir}/dist/geckoview/assets/omni.ja"
+
+    workingDir "${topobjdir}"
+
+    commandLine mozconfig.substs.GMAKE
+    args '-C'
+    args "${topobjdir}/mobile/android/base"
+    args 'gradle-omnijar'
+
+    // Only show the output if something went wrong.
+    ignoreExitValue = true
+    standardOutput = new ByteArrayOutputStream()
+    errorOutput = standardOutput
+    doLast {
+        if (execResult.exitValue != 0) {
+            throw new GradleException("Process '${commandLine}' finished with non-zero exit value ${execResult.exitValue}:\n\n${standardOutput.toString()}")
+        }
+    }
+}
+
 ext.configureVariantWithGeckoBinaries = { variant ->
     // :app needs the full Fennec omni.ja, whereas other projects need the
     // GeckoView-specific omni.ja.
     def omnijarDir = { "${topobjdir}/dist/${it}" }("app".equals(project.name) ? "fennec" : "geckoview")
     // All projects take the same Gecko libraries, which (for historical
     // reasons) are in "fennec".
     def distDir = "${topobjdir}/dist/fennec"
 
@@ -45,30 +78,21 @@ ext.configureVariantWithGeckoBinaries = 
     def syncAssetsFromDistDir = task("syncAssetsFromDistDirFor${variant.name.capitalize()}", type: Sync) {
         into("${project.buildDir}/moz.build/src/${variant.name}/assets")
         from("${distDir}/assets") {
             exclude 'omni.ja'
         }
     }
 
     // Local (read, not 'official') builds want to reflect developer changes to
-    // the omnijar sources, and (when compiling) to reflect developer changes to
-    // the native binaries.  To do this, the Gradle build calls out to the
+    // the Omnijar sources.  To do this, the Gradle build calls out to the
     // moz.build system, which can be re-entrant.  Official builds are driven by
     // the moz.build system and should never be re-entrant in this way.
     if (!mozconfig.substs.MOZILLA_OFFICIAL) {
-        if ("app".equals(project.name)) {
-            syncOmnijarFromDistDir.dependsOn rootProject.machStagePackageForFennec
-            syncLibsFromDistDir.dependsOn rootProject.machStagePackageForFennec
-            syncAssetsFromDistDir.dependsOn rootProject.machStagePackageForFennec
-        } else {
-            syncOmnijarFromDistDir.dependsOn rootProject.machStagePackageForGeckoview
-            syncLibsFromDistDir.dependsOn rootProject.machStagePackageForGeckoview
-            syncAssetsFromDistDir.dependsOn rootProject.machStagePackageForGeckoview
-        }
+        syncOmnijarFromDistDir.dependsOn buildOmnijars
     }
 
     def assetGenTask = tasks.findByName("generate${variant.name.capitalize()}Assets")
     if ((variant.productFlavors*.name).contains('withGeckoBinaries')) {
         assetGenTask.dependsOn syncOmnijarFromDistDir
         assetGenTask.dependsOn syncLibsFromDistDir
         assetGenTask.dependsOn syncAssetsFromDistDir