Backed out 4 changesets (bug 1509573) for unexpected bustages a=backout.
authorRazvan Maries <rmaries@mozilla.com>
Fri, 30 Nov 2018 01:40:06 +0200
changeset 507983 ecb2818bcd055bc5381d0702c955ee78a64dc41c
parent 507982 6453222232be364fb8ce3fd29b6cbcd480e5f2e3
child 508064 e18555e129e755fd2fa06e2760f05d4ebafe4187
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1509573
milestone65.0a1
backs outace2d2ada8eff0cc3505f1d6cc1f81da2bcf9179
059741f95cf4d4f680bea3eeadb714e567cd5413
36cf7b9ce095cca9ebc495fb44f23125c100efa7
d7e0661e34c223c23d04ef2083b2191f15c1d3c9
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 4 changesets (bug 1509573) for unexpected bustages a=backout. Backed out changeset ace2d2ada8ef (bug 1509573) Backed out changeset 059741f95cf4 (bug 1509573) Backed out changeset 36cf7b9ce095 (bug 1509573) Backed out changeset d7e0661e34c2 (bug 1509573)
build.gradle
config/faster/rules.mk
mobile/android/app/build.gradle
mobile/android/app/omnijar/build.gradle
mobile/android/base/Makefile.in
mobile/android/base/moz.build
mobile/android/gradle/with_gecko_binaries.gradle
--- a/build.gradle
+++ b/build.gradle
@@ -53,132 +53,54 @@ buildscript {
     if (gradle.mozconfig.substs.MOZ_ANDROID_GOOGLE_PLAY_SERVICES) {
         ext.google_play_services_version = '15.0.1'
     }
 
     dependencies {
         classpath 'org.mozilla.apilint:apilint:0.1.4'
         classpath 'com.android.tools.build:gradle:3.1.4'
         classpath 'com.getkeepsafe.dexcount:dexcount-gradle-plugin:0.8.2'
-        classpath 'org.apache.commons:commons-exec:1.3'
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
     }
 }
 
-// A stream that processes bytes line by line, prepending a tag before sending
-// each line to Gradle's logging.
-class TaggedLogOutputStream extends org.apache.commons.exec.LogOutputStream {
-    String tag
-    Logger logger
-
-    TaggedLogOutputStream(tag, logger) {
-        this.tag = tag
-        this.logger = logger
-    }
-
-    void processLine(String line, int level) {
-        logger.lifecycle("${this.tag} ${line}")
-    }
-}
-
-ext.geckoBinariesOnlyIf = { task ->
-    // Never for official builds.
-    if (mozconfig.substs.MOZILLA_OFFICIAL) {
-        rootProject.logger.lifecycle("Skipping task ${task.path} because: MOZILLA_OFFICIAL")
-        return false
-    }
-
+if ('multi' == System.env.AB_CD) {
     // Multi-l10n builds set `AB_CD=multi`, which isn't a valid locale.  This
     // causes the
     //
-    // |mach build| > |mach gradle| > AndroidManifest.xml > strings.xml > multi/brand.dtd
+    // |mach build| > |mach gradle| > |make gradle-targets| > AndroidManifest.xml > strings.xml > multi/brand.dtd
     //
     // dependency chain to fail, since multi isn't a real locale.  To avoid
     // this, if Gradle is invoked with AB_CD=multi, we don't invoke Make at all.
-    if ('multi' == System.env.AB_CD) {
-        rootProject.logger.lifecycle("Skipping task ${task.path} because: AB_CD=multi")
-        return false
-    }
-
+    task generateCodeAndResources()
+} else if (System.env.IS_LANGUAGE_REPACK == '1') {
     // Single-locale l10n repacks set `IS_LANGUAGE_REPACK=1` and handle resource
     // and code generation themselves.
-    if ('1' == System.env.IS_LANGUAGE_REPACK) {
-        rootProject.logger.lifecycle("Skipping task ${task.path} because: IS_LANGUAGE_REPACK")
-        return false
-    }
-
-    rootProject.logger.lifecycle("Executing task ${task.path}")
-    return true
-}
-
-task machBuildFaster(type: Exec) {
-    onlyIf rootProject.ext.geckoBinariesOnlyIf
-
-    workingDir "${topsrcdir}"
-
-    commandLine mozconfig.substs.PYTHON
-    args "${topsrcdir}/mach"
-    args 'build'
-    args 'faster'
-
-    // Add `-v` if we're running under `--info` (or `--debug`).
-    if (project.logger.isEnabled(LogLevel.INFO)) {
-        args '-v'
-    }
-
-    // `path` is like `:machBuildFaster`.
-    standardOutput = new TaggedLogOutputStream("${path}>", logger)
-    errorOutput = standardOutput
-}
-
-def createMachStagePackageTask(name) {
-    return task(name, type: Exec) {
-        onlyIf rootProject.ext.geckoBinariesOnlyIf
-
-        dependsOn rootProject.machBuildFaster
-
-        // We'd prefer to take these from the :omnijar project directly, but
-        // it's awkward to reach across projects at evaluation time, so we
-        // duplicate the list here.
-        inputs.dir "${topsrcdir}/mobile/android/chrome"
-        inputs.dir "${topsrcdir}/mobile/android/components"
-        inputs.dir "${topsrcdir}/mobile/android/locales"
-        inputs.dir "${topsrcdir}/mobile/android/modules"
-        inputs.dir "${topsrcdir}/mobile/android/themes"
-        inputs.dir "${topsrcdir}/toolkit"
-
+    task generateCodeAndResources()
+} else {
+    task generateCodeAndResources(type:Exec) {
         workingDir "${topobjdir}"
 
-        // We'd prefer this to be a `mach` invocation, but `mach build
-        // mobile/android/installer/stage-package` doesn't work as expected.
         commandLine mozconfig.substs.GMAKE
         args '-C'
-        args "${topobjdir}/mobile/android/installer"
-        args 'stage-package'
+        args "${topobjdir}/mobile/android/base"
+        args 'gradle-targets'
 
-        outputs.file "${topobjdir}/dist/fennec/assets/${mozconfig.substs.ANDROID_CPU_ARCH}/libxul.so"
-        outputs.file "${topobjdir}/dist/fennec/lib/${mozconfig.substs.ANDROID_CPU_ARCH}/libmozglue.so"
-
-        // `path` is like `:machStagePackage`.
-        standardOutput = new TaggedLogOutputStream("${path}>", logger)
+        // Only show the output if something went wrong.
+        ignoreExitValue = true
+        standardOutput = new ByteArrayOutputStream()
         errorOutput = standardOutput
+        doLast {
+            if (execResult.exitValue != 0) {
+                throw new GradleException("Process '${commandLine}' finished with non-zero exit value ${execResult.exitValue}:\n\n${standardOutput.toString()}")
+            }
+        }
     }
 }
 
-createMachStagePackageTask("machStagePackageForFennec").with {
-    outputs.file "${topobjdir}/dist/fennec/assets/omni.ja"
-}
-
-createMachStagePackageTask("machStagePackageForGeckoview").with {
-    args 'MOZ_GECKOVIEW_JAR=1'
-    outputs.file "${topobjdir}/dist/geckoview/assets/omni.ja"
-    // Avoid races between stage-package invocations.
-    mustRunAfter tasks["machStagePackageForFennec"]
-}
-
 afterEvaluate {
     subprojects { project ->
         if (project.name != 'thirdparty') {
             tasks.withType(JavaCompile) {
                 // Add compiler args for all code except third-party code.
                 options.compilerArgs += [
                     // Turn on all warnings, except...
                     "-Xlint:all",
@@ -204,20 +126,20 @@ afterEvaluate {
                 }
             }
         }
 
         if (!hasProperty('android')) {
             return
         }
         android.applicationVariants.all {
-            preBuild.dependsOn rootProject.machBuildFaster
+            preBuild.dependsOn rootProject.generateCodeAndResources
         }
         android.libraryVariants.all {
-            preBuild.dependsOn rootProject.machBuildFaster
+            preBuild.dependsOn rootProject.generateCodeAndResources
         }
     }
 }
 
 apply plugin: 'idea'
 
 idea {
     project {
--- a/config/faster/rules.mk
+++ b/config/faster/rules.mk
@@ -61,23 +61,16 @@ endif
 # Extra define to trigger some workarounds. We should strive to limit the
 # use of those. As of writing the only one is in browser/locales/jar.mn.
 ACDEFINES += -DBUILD_FASTER
 
 # Files under the faster/ sub-directory, however, are not meant to use the
 # fallback
 $(TOPOBJDIR)/faster/%: ;
 
-ifeq ($(MOZ_BUILD_APP),mobile/android)
-# The generic rule doesn't handle relative directories, which are used
-# extensively in mobile/android/base.
-$(TOPOBJDIR)/mobile/android/base/% : $(TOPOBJDIR)/buildid.h FORCE
-	$(MAKE) -C $(TOPOBJDIR)/mobile/android/base $*
-endif
-
 # Generic rule to fall back to the recursive make backend.
 # This needs to stay after other $(TOPOBJDIR)/* rules because GNU Make
 # <3.82 apply pattern rules in definition order, not stem length like
 # modern GNU Make.
 $(TOPOBJDIR)/%: FORCE
 	$(MAKE) -C $(dir $@) $(notdir $@)
 
 # Install files using install manifests
--- a/mobile/android/app/build.gradle
+++ b/mobile/android/app/build.gradle
@@ -333,17 +333,17 @@ android.applicationVariants.all { varian
     // `sourceSets` task, for reasons unknown.
     variant.registerGeneratedResFolders(project.files(syncPreprocessedRes.destinationDir).builtBy(syncPreprocessedRes))
 
     // It's not easy -- see the backout in Bug 1242213 -- to change the
     // <manifest> package for Fennec.  Gradle has grown a mechanism to achieve
     // what we want for Fennec, however, with applicationId.  To use the same
     // manifest as moz.build, we replace the package with org.mozilla.gecko (the
     // eventual package) here.
-    def rewriteManifestPackage = task("rewriteManifestPackageFor${variant.name.capitalize()}", type: Copy, dependsOn: rootProject.machBuildFaster) {
+    def rewriteManifestPackage = task("rewriteManifestPackageFor${variant.name.capitalize()}", type: Copy, dependsOn: rootProject.generateCodeAndResources) {
         into("${project.buildDir}/moz.build/src/${variant.name}")
         from("${topobjdir}/mobile/android/base/AndroidManifest.xml")
         filter { it.replaceFirst(/package=".*?"/, 'package="org.mozilla.gecko"') }
         exclude('**/*.mkdir.done')
     }
 
     // Every configuration needs the stub manifest at
     // src/main/AndroidManifest.xml and the generated manifest.  We can't use
@@ -354,19 +354,19 @@ android.applicationVariants.all { varian
 
 
     // Local (read, not 'official') builds want to reflect developer changes to
     // AndroidManifest.xml.in, strings.xml, and preprocessed Java code.  To do
     // this, the Gradle build calls out to the moz.build system, which can be
     // re-entrant.  Official builds are driven by the moz.build system and
     // should never be re-entrant in this way.
     if (!mozconfig.substs.MOZILLA_OFFICIAL) {
-        syncPreprocessedJava.dependsOn rootProject.machBuildFaster
-        syncPreprocessedRes.dependsOn rootProject.machBuildFaster
-        rewriteManifestPackage.dependsOn rootProject.machBuildFaster
+        syncPreprocessedJava.dependsOn rootProject.generateCodeAndResources
+        syncPreprocessedRes.dependsOn rootProject.generateCodeAndResources
+        rewriteManifestPackage.dependsOn rootProject.generateCodeAndResources
     }
 
     // When driven from moz.build via |mach build|, Gradle does not require or
     // use Gecko binaries.  It's only |mach package| that packs the Gecko
     // binaries into the resulting APK.  The "withoutGeckoBinaries" variants
     // handle this.  When driven from Android Studio or Gradle, the
     // "withGeckoBinaries" variants handle packing the Gecko binaries into the
     // resulting APK (for on-device deployment).  They also update the Omnijars
--- a/mobile/android/app/omnijar/build.gradle
+++ b/mobile/android/app/omnijar/build.gradle
@@ -1,25 +1,24 @@
 buildDir "${topobjdir}/gradle/build/mobile/android/omnijar"
 
 apply plugin: 'java'
 
-// This project is a dummy project; the JAR produced is not used.  The root
+// This project is a dummy project; the JAR produced is not used.  The :app
 // project uses the set of inputs here to check if the omnijar needs to be
 // rebuilt.  By listing them here as resource directories, IntelliJ labels each
 // checked directly nicely.  Why list the directories here?  There's a mismatch
 // between SourceDirectorySet and TaskInputs: the former is directory oriented,
 // while the latter is more general.  That means its easy to convert this list
-// into inputs for the root project, but not vice-versa.  Sadly this would mean
-// that the root project evaluation depends on :omnijar, so we duplicate in the
-// root project.
+// into inputs for :app, but not vice-versa.  Sadly this implies that :app
+// evaluation depends on :omnijar, but the evaluation overhead is low enough
+// that we accept it.
 sourceSets {
     main {
-        // These are supposed to be files that go into the omnijar.  But yes,
-        // toolkit/ has lots of native code.
+        // Depend on the Gecko resources in mobile/android.
         resources {
             srcDir "${topsrcdir}/mobile/android/chrome"
             srcDir "${topsrcdir}/mobile/android/components"
             srcDir "${topsrcdir}/mobile/android/locales"
             srcDir "${topsrcdir}/mobile/android/modules"
             srcDir "${topsrcdir}/mobile/android/themes"
             srcDir "${topsrcdir}/toolkit"
         }
--- a/mobile/android/base/Makefile.in
+++ b/mobile/android/base/Makefile.in
@@ -1,20 +1,65 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # Developer builds call mach -> Make -> gradle -> mach, which races to
 # find and create .mozconfig files and to generate targets.
 .NOTPARALLEL:
 
+generated_resources := \
+  AndroidManifest.xml \
+  res/raw/suggestedsites.json \
+  res/values/strings.xml \
+  $(NULL)
+
+generated_files := \
+  AndroidManifest.xml \
+  generated/preprocessed/org/mozilla/gecko/AdjustConstants.java \
+  generated/preprocessed/org/mozilla/gecko/AppConstants.java \
+  generated/preprocessed/org/mozilla/gecko/MmaConstants.java \
+  $(NULL)
+
 include $(topsrcdir)/config/AB_rCD.mk
 
 chrome-%:: AB_CD=$*
 chrome-%::
 	$(MAKE) \
 	  res/values$(AB_rCD)/strings.xml \
 	  res/raw$(AB_rCD)/suggestedsites.json \
 	  AB_CD=$*
 
 include $(topsrcdir)/config/rules.mk
 
+# This target is only used by IDE integrations. It rebuilds resources
+# that end up in omni.ja using the equivalent of |mach build faster|,
+# does most of the packaging step, and then updates omni.ja in
+# place. If you're not using an IDE, you should be using |mach build
+# mobile/android && mach package|.
+$(ABS_DIST)/fennec/$(OMNIJAR_NAME): FORCE
+	$(REPORT_BUILD)
+	$(MAKE) -C ../../../faster
+	$(MAKE) -C ../installer stage-package
+	$(MKDIR) -p $(@D)
+	rsync --update $(DIST)/fennec/$(notdir $(OMNIJAR_NAME)) $@
+	$(RM) $(DIST)/fennec/$(notdir $(OMNIJAR_NAME))
+
+ifndef MOZILLA_OFFICIAL
+# Targets built very early during a Gradle build.  In automation,
+# these are built before Gradle is invoked, and gradle-targets is not
+# made at all.  This is required to avoid building gradle-targets with
+# AB_CD=multi during multi-l10n builds.
+gradle-targets: $(generated_resources) $(generated_files)
+
+# Local developers update omni.ja during their builds.  There's a
+# chicken-and-egg problem here.
+gradle-omnijar: $(abspath $(DIST)/fennec/$(OMNIJAR_NAME))
+else
+# In automation, omni.ja is built only during packaging.
+gradle-omnijar:
+
+gradle-targets:
+endif
+
+.PHONY: gradle-targets gradle-omnijar
+
 export:: android_apks
--- a/mobile/android/base/moz.build
+++ b/mobile/android/base/moz.build
@@ -178,38 +178,31 @@ for f in ['res/values/strings.xml',
     strings.script = '/python/mozbuild/mozbuild/action/generate_strings_xml.py'
     strings.inputs = [
         'strings.xml.in',
         # The `locales/en-US/` will be rewritten to the locale-specific path.
         'locales/en-US/android_strings.dtd',
         'locales/en-US/sync_strings.dtd',
     ]
 
-generated_inputs = [
+# The recursive make backend treats the first output specially: it's passed as
+# an open FileAvoidWrite to the invoked script.  That doesn't work well with
+# the Gradle task that generates all of the outputs, so we add a dummy first
+# output.
+t = ('android_apks',
+     CONFIG['GRADLE_ANDROID_APP_APK'],
+     CONFIG['GRADLE_ANDROID_APP_ANDROIDTEST_APK'])
+
+GENERATED_FILES += [t]
+GENERATED_FILES[t].force = True
+GENERATED_FILES[t].script = '/mobile/android/gradle.py:assemble_app'
+GENERATED_FILES[t].inputs += [
     '!AndroidManifest.xml',
     '!generated/preprocessed/org/mozilla/gecko/AdjustConstants.java',
     '!generated/preprocessed/org/mozilla/gecko/AppConstants.java',
     '!generated/preprocessed/org/mozilla/gecko/CrashReporterConstants.java',
     '!generated/preprocessed/org/mozilla/gecko/MmaConstants.java',
     # These all depend on AB_CD, which isn't captured in this definition.  Due
     # to subtle RecursiveMake details, everything works out.  In the future we
     # can try to express the APKs themselves as LOCALIZED_GENERATED_FILES.
     '!res/raw/suggestedsites.json',
     '!res/values/strings.xml',
 ]
-
-# The recursive make backend treats the first output specially: it's passed as
-# an open FileAvoidWrite to the invoked script.  That doesn't work well with
-# the Gradle task that generates all of the outputs, so we add a dummy first
-# output.
-t = ('android_apks',
-     CONFIG['GRADLE_ANDROID_APP_APK'],
-     CONFIG['GRADLE_ANDROID_APP_ANDROIDTEST_APK'])
-
-GENERATED_FILES += [t]
-GENERATED_FILES[t].force = True
-GENERATED_FILES[t].script = '/mobile/android/gradle.py:assemble_app'
-GENERATED_FILES[t].inputs += generated_inputs
-
-# This makes |mach build faster| build these files.  See the workaround in
-# config/faster/rules.mk.  None of these files appear in package-manifest.in,
-# so they're not packaged.
-FINAL_TARGET_FILES.android += generated_inputs
--- a/mobile/android/gradle/with_gecko_binaries.gradle
+++ b/mobile/android/gradle/with_gecko_binaries.gradle
@@ -1,16 +1,49 @@
 /* -*- Mode: Groovy; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+// The omnijar inputs are listed as resource directory inputs to a dummy JAR.
+// That arrangement labels them nicely in IntelliJ.  See the comment in the
+// :omnijar project for more context.
+evaluationDependsOn(':omnijar')
+
 // The JNI wrapper generation tasks depend on the JAR creation task of the :annotations project.
 evaluationDependsOn(':annotations')
 
+task buildOmnijars(type:Exec) {
+    dependsOn rootProject.generateCodeAndResources
+
+    // See comment in :omnijar project regarding interface mismatches here.
+    inputs.file(project(':omnijar').sourceSets.main.resources.srcDirs).skipWhenEmpty() 
+
+    // Produce both the Fennec and the GeckoView omnijars.
+    outputs.file "${topobjdir}/dist/fennec/assets/omni.ja"
+    outputs.file "${topobjdir}/dist/geckoview/assets/omni.ja"
+
+    workingDir "${topobjdir}"
+
+    commandLine mozconfig.substs.GMAKE
+    args '-C'
+    args "${topobjdir}/mobile/android/base"
+    args 'gradle-omnijar'
+
+    // Only show the output if something went wrong.
+    ignoreExitValue = true
+    standardOutput = new ByteArrayOutputStream()
+    errorOutput = standardOutput
+    doLast {
+        if (execResult.exitValue != 0) {
+            throw new GradleException("Process '${commandLine}' finished with non-zero exit value ${execResult.exitValue}:\n\n${standardOutput.toString()}")
+        }
+    }
+}
+
 ext.configureVariantWithGeckoBinaries = { variant ->
     // :app needs the full Fennec omni.ja, whereas other projects need the
     // GeckoView-specific omni.ja.
     def omnijarDir = { "${topobjdir}/dist/${it}" }("app".equals(project.name) ? "fennec" : "geckoview")
     // All projects take the same Gecko libraries, which (for historical
     // reasons) are in "fennec".
     def distDir = "${topobjdir}/dist/fennec"
 
@@ -45,30 +78,21 @@ ext.configureVariantWithGeckoBinaries = 
     def syncAssetsFromDistDir = task("syncAssetsFromDistDirFor${variant.name.capitalize()}", type: Sync) {
         into("${project.buildDir}/moz.build/src/${variant.name}/assets")
         from("${distDir}/assets") {
             exclude 'omni.ja'
         }
     }
 
     // Local (read, not 'official') builds want to reflect developer changes to
-    // the omnijar sources, and (when compiling) to reflect developer changes to
-    // the native binaries.  To do this, the Gradle build calls out to the
+    // the Omnijar sources.  To do this, the Gradle build calls out to the
     // moz.build system, which can be re-entrant.  Official builds are driven by
     // the moz.build system and should never be re-entrant in this way.
     if (!mozconfig.substs.MOZILLA_OFFICIAL) {
-        if ("app".equals(project.name)) {
-            syncOmnijarFromDistDir.dependsOn rootProject.machStagePackageForFennec
-            syncLibsFromDistDir.dependsOn rootProject.machStagePackageForFennec
-            syncAssetsFromDistDir.dependsOn rootProject.machStagePackageForFennec
-        } else {
-            syncOmnijarFromDistDir.dependsOn rootProject.machStagePackageForGeckoview
-            syncLibsFromDistDir.dependsOn rootProject.machStagePackageForGeckoview
-            syncAssetsFromDistDir.dependsOn rootProject.machStagePackageForGeckoview
-        }
+        syncOmnijarFromDistDir.dependsOn buildOmnijars
     }
 
     def assetGenTask = tasks.findByName("generate${variant.name.capitalize()}Assets")
     if ((variant.productFlavors*.name).contains('withGeckoBinaries')) {
         assetGenTask.dependsOn syncOmnijarFromDistDir
         assetGenTask.dependsOn syncLibsFromDistDir
         assetGenTask.dependsOn syncAssetsFromDistDir