Bug 1553065 - Move profile-use mozconfig info into common mozconfigs; r=firefox-build-system-reviewers,chmanchester a=RyanVM
authorMike Shal <mshal@mozilla.com>
Fri, 31 May 2019 23:56:16 +0000
changeset 533887 4f6a9270754a0ca4e368bb0bf3256c72669fe79c
parent 533886 4a7962ae170f16bc2b7b0e50348322ca1f01a7dc
child 533888 9b37433a054508d16843307ee2b0d901e23720fa
push id11452
push userjlorenzo@mozilla.com
push dateFri, 14 Jun 2019 15:11:56 +0000
treeherdermozilla-beta@4f6a9270754a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfirefox-build-system-reviewers, chmanchester, RyanVM
bugs1553065
milestone68.0
Bug 1553065 - Move profile-use mozconfig info into common mozconfigs; r=firefox-build-system-reviewers,chmanchester a=RyanVM The 3-tier PGO builds used a separate mozconfig called 'profile-use' for the final tier. This created a problem when it rode to beta, since the same mozconfig was used for all trees, which meant we ended up with nightly branding on beta builds. With the PGO-enabling logic in common mozconfigs, we can enable it by setting the MOZ_PGO_PROFILE_USE environment variable from the task definition. All of the final-tier PGO builds now use the nightly, beta, etc mozconfigs like before, so branding should be intact. Differential Revision: https://phabricator.services.mozilla.com/D33172
browser/config/mozconfigs/linux64/profile-use
build/unix/mozconfig.unix
mobile/android/config/mozconfigs/android-aarch64/beta-profile-use
mobile/android/config/mozconfigs/android-aarch64/profile-use
mobile/android/config/mozconfigs/android-api-16/profile-use
mobile/android/config/mozconfigs/common
taskcluster/ci/build/android.yml
taskcluster/ci/build/linux.yml
taskcluster/taskgraph/transforms/build.py
testing/mozharness/configs/builds/releng_sub_android_configs/64_aarch64_beta_pgo.py
testing/mozharness/configs/builds/releng_sub_android_configs/64_aarch64_pgo.py
testing/mozharness/configs/builds/releng_sub_android_configs/64_api_16_profile_use.py
deleted file mode 100644
--- a/browser/config/mozconfigs/linux64/profile-use
+++ /dev/null
@@ -1,6 +0,0 @@
-. $topsrcdir/browser/config/mozconfigs/linux64/beta
-
-export MOZ_LTO=1
-ac_add_options --enable-profile-use
-ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log
-ac_add_options --with-pgo-profile-path=/builds/worker/fetches/default.profraw
--- a/build/unix/mozconfig.unix
+++ b/build/unix/mozconfig.unix
@@ -21,16 +21,21 @@ else
       export CFLAGS="$CFLAGS -fcrash-diagnostics-dir=${UPLOAD_PATH}"
       export CXXFLAGS="$CXXFLAGS -fcrash-diagnostics-dir=${UPLOAD_PATH}"
       ;;
     esac
 
     if [ -n "$MOZ_PGO" ]; then
         if [ -z "$USE_ARTIFACT" ]; then
             export MOZ_LTO=1
+            if [ -n "$MOZ_PGO_PROFILE_USE" ]; then
+                ac_add_options --enable-profile-use
+                ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log
+                ac_add_options --with-pgo-profile-path=/builds/worker/fetches
+            fi
         fi
         export LLVM_PROFDATA="$TOOLTOOL_DIR/clang/bin/llvm-profdata"
         export AR="$topsrcdir/clang/bin/llvm-ar"
         export NM="$topsrcdir/clang/bin/llvm-nm"
         export RANLIB="$topsrcdir/clang/bin/llvm-ranlib"
     fi
 
     mk_add_options "export PATH=$TOOLTOOL_DIR/binutils/bin:$PATH"
deleted file mode 100644
--- a/mobile/android/config/mozconfigs/android-aarch64/beta-profile-use
+++ /dev/null
@@ -1,10 +0,0 @@
-. "$topsrcdir/mobile/android/config/mozconfigs/android-aarch64/beta"
-
-export LLVM_PROFDATA="$topsrcdir/clang/bin/llvm-profdata"
-
-ac_add_options --enable-profile-use
-
-# This is disabled because jarlog re-ordering breaks apk publishing tasks,
-# see bug 1539933.
-# ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log
-ac_add_options --with-pgo-profile-path=/builds/worker/fetches/default.profraw
deleted file mode 100644
--- a/mobile/android/config/mozconfigs/android-aarch64/profile-use
+++ /dev/null
@@ -1,10 +0,0 @@
-. "$topsrcdir/mobile/android/config/mozconfigs/android-aarch64/nightly"
-
-export LLVM_PROFDATA="$topsrcdir/clang/bin/llvm-profdata"
-
-ac_add_options --enable-profile-use
-
-# This is disabled because jarlog re-ordering breaks apk publishing tasks,
-# see bug 1539933.
-# ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log
-ac_add_options --with-pgo-profile-path=/builds/worker/fetches/default.profraw
deleted file mode 100644
--- a/mobile/android/config/mozconfigs/android-api-16/profile-use
+++ /dev/null
@@ -1,11 +0,0 @@
-. "$topsrcdir/mobile/android/config/mozconfigs/android-api-16/nightly"
-
-export LLVM_PROFDATA="$topsrcdir/clang/bin/llvm-profdata"
-
-ac_add_options --enable-profile-use
-
-# This is disabled because jarlog re-ordering breaks apk publishing tasks,
-# see bug 1539933.
-# ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log
-
-ac_add_options --with-pgo-profile-path=/builds/worker/fetches/default.profraw
--- a/mobile/android/config/mozconfigs/common
+++ b/mobile/android/config/mozconfigs/common
@@ -84,12 +84,23 @@ if test "$MOZ_UPDATE_CHANNEL" = "release
 elif test "$MOZ_UPDATE_CHANNEL" = "beta" ; then
     ac_add_options --with-pocket-api-keyfile=/builds/pocket-api-beta.token
 elif test "$MOZ_UPDATE_CHANNEL" = "nightly" ; then
     ac_add_options --with-pocket-api-keyfile=/builds/pocket-api-nightly.token
 else
     ac_add_options --with-pocket-api-keyfile="$topsrcdir/mobile/android/base/pocket-api-sandbox.token"
 fi
 
+if [ -n "$MOZ_PGO_PROFILE_USE" ]; then
+    export LLVM_PROFDATA="$topsrcdir/clang/bin/llvm-profdata"
+
+    ac_add_options --enable-profile-use
+
+    # This is disabled because jarlog re-ordering breaks apk publishing tasks,
+    # see bug 1539933.
+    # ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log
+    ac_add_options --with-pgo-profile-path=/builds/worker/fetches
+fi
+
 # Package js shell.
 export MOZ_PACKAGE_JSSHELL=1
 
 JS_BINARY="$topsrcdir/mobile/android/config/js_wrapper.sh"
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -248,17 +248,17 @@ android-api-16/pgo:
         symbol: Bpgo(B)
     worker-type: b-linux
     worker:
         env:
             TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest"
     run:
         actions: [get-secrets, build]
         config: ["builds/releng_base_android_64_builds.py"]
-        custom-build-variant-cfg: api-16-profile-use
+        custom-build-variant-cfg: api-16
 
 android-api-16-without-google-play-services/opt:
     description: "Android 4.0 api-16+ (without Google Play Services) Opt"
     index:
         job-name: android-api-16-without-google-play-services-opt
     treeherder:
         platform: android-4-0-armv7-api16/opt
         symbol: BnoGPS
@@ -294,22 +294,22 @@ android-api-16-nightly/opt:
     worker:
         env:
             TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest"
     run:
         actions: [get-secrets, build, multi-l10n]
         config:
             - builds/releng_base_android_64_builds.py
             - taskcluster_nightly.py
-        custom-build-variant-cfg: api-16-profile-use
+        custom-build-variant-cfg: api-16
         update-channel: nightly
 
 android-api-16-beta/opt:
     description: "Android 4.0 api-16+ Beta"
-    use-pgo: android-api-16/pgo
+    use-pgo: android-api-16-beta/pgo
     attributes:
         nightly: true
         enable-full-crashsymbols: true
         release-type: beta
     shipping-phase: build
     shipping-product: fennec
     index:
         job-name: android-api-16-beta-opt
@@ -330,17 +330,17 @@ android-api-16-beta/opt:
         config:
             - builds/releng_base_android_64_builds.py
             - taskcluster_nightly.py
         custom-build-variant-cfg: api-16-beta
         update-channel: beta
 
 android-api-16-beta-test/opt:
     description: "Android 4.0 api-16+ Beta"
-    use-pgo: android-api-16/pgo
+    use-pgo: android-api-16-beta/pgo
     attributes:
         enable-full-crashsymbols: true
         release-type: beta
     shipping-phase: build
     shipping-product: fennec
     index:
         job-name: android-api-16-beta-opt
     treeherder:
@@ -388,17 +388,17 @@ android-aarch64/pgo:
         symbol: Bpgo(B)
     worker-type: b-linux
     worker:
         env:
             TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest"
     run:
         actions: [get-secrets, build]
         config: ["builds/releng_base_android_64_builds.py"]
-        custom-build-variant-cfg: aarch64-pgo
+        custom-build-variant-cfg: aarch64
 
 android-aarch64/debug:
     description: "Android 5.0 AArch64 Debug"
     index:
         job-name: android-aarch64-debug
     attributes:
         enable-full-crashsymbols: true
     treeherder:
@@ -457,17 +457,17 @@ android-aarch64-nightly/opt:
     worker:
         env:
             TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest"
     run:
         actions: [get-secrets, build, multi-l10n]
         config:
             - builds/releng_base_android_64_builds.py
             - taskcluster_nightly.py
-        custom-build-variant-cfg: aarch64-pgo
+        custom-build-variant-cfg: aarch64
         update-channel: nightly
 
 android-aarch64-beta/opt:
     description: "Android 5.0 AArch64 Beta"
     use-pgo: android-api-16-beta/pgo
     attributes:
         nightly: true
         enable-full-crashsymbols: true
@@ -488,17 +488,17 @@ android-aarch64-beta/opt:
             - name: public/build/target.maven.zip
               path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/target.maven.zip
               type: file
     run:
         actions: [get-secrets, build, multi-l10n]
         config:
             - builds/releng_base_android_64_builds.py
             - taskcluster_nightly.py
-        custom-build-variant-cfg: aarch64-beta-pgo
+        custom-build-variant-cfg: aarch64-beta
         update-channel: beta
 
 android-x86_64/opt:
     description: "Android 5.0 x86-64 Opt"
     index:
         job-name: android-x86_64-opt
     attributes:
         enable-full-crashsymbols: true
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -825,17 +825,16 @@ linux64-shippable/opt:
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
             - taskcluster_nightly.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: public
         need-xvfb: true
-        mozconfig-variant: profile-use
     toolchains:
         - linux64-binutils
         - linux64-clang
         - linux64-rust
         - linux64-rust-size
         - linux64-cbindgen
         - linux64-nasm
         - linux64-node
--- a/taskcluster/taskgraph/transforms/build.py
+++ b/taskcluster/taskgraph/transforms/build.py
@@ -113,16 +113,17 @@ def use_profile_data(config, jobs):
         # task to use in the use_pgo field.
         if use_pgo is True:
             name = job['name']
         else:
             name = use_pgo
         dependencies = 'generate-profile-{}'.format(name)
         job.setdefault('dependencies', {})['generate-profile'] = dependencies
         job.setdefault('fetches', {})['generate-profile'] = ['profdata.tar.xz']
+        job['worker']['env'].update({"MOZ_PGO_PROFILE_USE": "1"})
         yield job
 
 
 @transforms.add
 def set_env(config, jobs):
     """Set extra environment variables from try command line."""
     env = []
     if config.params['try_mode'] == 'try_option_syntax':
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_android_configs/64_aarch64_beta_pgo.py
+++ /dev/null
@@ -1,5 +0,0 @@
-config = {
-    'stage_platform': 'android-aarch64-pgo',
-    'src_mozconfig': 'mobile/android/config/mozconfigs/android-aarch64/beta-profile-use',
-    'multi_locale_config_platform': 'android',
-}
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_android_configs/64_aarch64_pgo.py
+++ /dev/null
@@ -1,5 +0,0 @@
-config = {
-    'stage_platform': 'android-aarch64-pgo',
-    'src_mozconfig': 'mobile/android/config/mozconfigs/android-aarch64/profile-use',
-    'multi_locale_config_platform': 'android',
-}
deleted file mode 100644
--- a/testing/mozharness/configs/builds/releng_sub_android_configs/64_api_16_profile_use.py
+++ /dev/null
@@ -1,5 +0,0 @@
-config = {
-    'stage_platform': 'android-api-16',
-    'src_mozconfig': 'mobile/android/config/mozconfigs/android-api-16/profile-use',
-    'multi_locale_config_platform': 'android',
-}