Bug 1517532 - Extend artifact builds to find and download artifacts from pgo builds when MOZ_PGO is set. r=nalexander
authorChris Manchester <cmanchester@mozilla.com>
Tue, 08 Jan 2019 23:03:27 +0000
changeset 510072 29429b3bef3eda44f50a2efd8361987286c3276a
parent 510071 668c6d9c4fe9e21d036d24b6e1cc152b3bd41805
child 510073 a13bd50f3cd449db176f17e6faecfe74c0238fc8
push id10547
push userffxbld-merge
push dateMon, 21 Jan 2019 13:03:58 +0000
treeherdermozilla-beta@24ec1916bffe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1517532
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1517532 - Extend artifact builds to find and download artifacts from pgo builds when MOZ_PGO is set. r=nalexander Differential Revision: https://phabricator.services.mozilla.com/D15988
Makefile.in
build/moz.configure/pgo.configure
build/moz.configure/toolchain.configure
build/mozconfig.artifact.automation
moz.configure
python/mozbuild/mozbuild/artifacts.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -193,22 +193,24 @@ profiledbuild::
 	$(MAKE) maybe_clobber_profiledbuild
 	$(call BUILDSTATUS,TIER_FINISH pgo_clobber)
 	$(call BUILDSTATUS,TIER_START pgo_profile_use)
 	$(MAKE) default MOZ_PROFILE_USE=1 $(if $(CLANG_CL),MOZ_PROFILE_ORDER_FILE=$(topobjdir)/cygprofile.txt)
 	$(call BUILDSTATUS,TIER_FINISH pgo_profile_use)
 
 # Change default target to PGO build if PGO is enabled.
 ifdef MOZ_PGO
+ifdef COMPILE_ENVIRONMENT
 # If one of these is already set in addition to PGO we are doing a single phase
 # of PGO in isolation, so don't override the default target.
 ifeq (,$(MOZ_PROFILE_GENERATE)$(MOZ_PROFILE_USE))
 OVERRIDE_DEFAULT_GOAL := profiledbuild
 endif
 endif
+endif
 
 include $(topsrcdir)/config/rules.mk
 
 ifdef SCCACHE_VERBOSE_STATS
 default::
 	-$(CCACHE) --show-stats --stats-format=json > sccache-stats.json
 	@echo "===SCCACHE STATS==="
 	-$(CCACHE) --show-stats
new file mode 100644
--- /dev/null
+++ b/build/moz.configure/pgo.configure
@@ -0,0 +1,58 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+llvm_profdata = check_prog('LLVM_PROFDATA', ['llvm-profdata'],
+                           allow_missing=True)
+
+add_old_configure_assignment('LLVM_PROFDATA', llvm_profdata)
+
+# PGO
+# ==============================================================
+js_option('--enable-profile-generate',
+          help='Build a PGO instrumented binary')
+
+imply_option('MOZ_PGO',
+             depends_if('--enable-profile-generate')(lambda _: True))
+
+set_config('MOZ_PROFILE_GENERATE',
+           depends_if('--enable-profile-generate')(lambda _: True))
+
+js_option('--enable-profile-use',
+          help='Use a generated profile during the build')
+
+js_option('--with-pgo-profile-path',
+          help='Path to the (unmerged) profile path to use during the build',
+          nargs=1)
+
+imply_option('MOZ_PGO',
+             depends_if('--enable-profile-use')(lambda _: True))
+
+set_config('MOZ_PROFILE_USE',
+           depends_if('--enable-profile-use')(lambda _: True))
+
+js_option(env='MOZ_PGO', help='Build with profile guided optimizations')
+
+set_config('MOZ_PGO', depends('MOZ_PGO')(lambda x: bool(x)))
+add_old_configure_assignment('MOZ_PGO', depends('MOZ_PGO')(lambda x: bool(x)))
+
+@depends('--with-pgo-profile-path', '--enable-profile-use', 'LLVM_PROFDATA')
+def pgo_profile_path(path, pgo_use, profdata):
+    if not path:
+        return
+    if path and not pgo_use:
+        die('Pass --enable-profile-use to use --with-pgo-profile-path.')
+    if path and not profdata:
+        die('LLVM_PROFDATA must be set to process the pgo profile.')
+    return path[0]
+
+set_config('PGO_PROFILE_PATH', pgo_profile_path)
+
+option('--with-pgo-jarlog',
+       help='Use the provided jarlog file when packaging during a profile-use '
+            'build',
+       nargs=1)
+
+set_config('PGO_JARLOG_PATH', depends_if('--with-pgo-jarlog')(lambda p: p))
--- a/build/moz.configure/toolchain.configure
+++ b/build/moz.configure/toolchain.configure
@@ -14,72 +14,19 @@ js_option('--enable-release',
 
 
 @depends('--enable-release')
 def developer_options(value):
     if not value:
         return True
 
 
-llvm_profdata = check_prog('LLVM_PROFDATA', ['llvm-profdata'],
-                           allow_missing=True)
-
-add_old_configure_assignment('LLVM_PROFDATA', llvm_profdata)
-
-
 add_old_configure_assignment('DEVELOPER_OPTIONS', developer_options)
 set_config('DEVELOPER_OPTIONS', developer_options)
 
-# PGO
-# ==============================================================
-js_option('--enable-profile-generate',
-          help='Build a PGO instrumented binary')
-
-imply_option('MOZ_PGO',
-             depends_if('--enable-profile-generate')(lambda _: True))
-
-set_config('MOZ_PROFILE_GENERATE',
-           depends_if('--enable-profile-generate')(lambda _: True))
-
-js_option('--enable-profile-use',
-          help='Use a generated profile during the build')
-
-js_option('--with-pgo-profile-path',
-          help='Path to the (unmerged) profile path to use during the build',
-          nargs=1)
-
-imply_option('MOZ_PGO',
-             depends_if('--enable-profile-use')(lambda _: True))
-
-set_config('MOZ_PROFILE_USE',
-           depends_if('--enable-profile-use')(lambda _: True))
-
-js_option(env='MOZ_PGO', help='Build with profile guided optimizations')
-
-set_config('MOZ_PGO', depends('MOZ_PGO')(lambda x: bool(x)))
-add_old_configure_assignment('MOZ_PGO', depends('MOZ_PGO')(lambda x: bool(x)))
-
-@depends('--with-pgo-profile-path', '--enable-profile-use', 'LLVM_PROFDATA')
-def pgo_profile_path(path, pgo_use, profdata):
-    if not path:
-        return
-    if path and not pgo_use:
-        die('Pass --enable-profile-use to use --with-pgo-profile-path.')
-    if path and not profdata:
-        die('LLVM_PROFDATA must be set to process the pgo profile.')
-    return path[0]
-
-set_config('PGO_PROFILE_PATH', pgo_profile_path)
-
-option('--with-pgo-jarlog',
-       help='Use the provided jarlog file when packaging during a profile-use '
-            'build',
-       nargs=1)
-
-set_config('PGO_JARLOG_PATH', depends_if('--with-pgo-jarlog')(lambda p: p))
 
 # Code optimization
 # ==============================================================
 
 js_option('--disable-optimize',
           nargs='?',
           help='Disable optimizations via compiler flags')
 
--- a/build/mozconfig.artifact.automation
+++ b/build/mozconfig.artifact.automation
@@ -1,7 +1,8 @@
 # Common options for artifact builds to set automation steps.
 # This gets included before mozconfig.automation.
 
 MOZ_AUTOMATION_BUILD_SYMBOLS=0
 MOZ_AUTOMATION_L10N_CHECK=0
 MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0
 MOZ_AUTOMATION_ARTIFACT_BUILDS=1
+USE_ARTIFACT=1
--- a/moz.configure
+++ b/moz.configure
@@ -114,16 +114,17 @@ js_option('--enable-rust-debug',
 
 @depends(when='--enable-rust-debug')
 def debug_rust():
     return True
 
 set_config('MOZ_DEBUG_RUST', debug_rust)
 set_define('MOZ_DEBUG_RUST', debug_rust)
 
+include('build/moz.configure/pgo.configure')
 include('build/moz.configure/pkg.configure')
 # Make this assignment here rather than in pkg.configure to avoid
 # requiring this file in unit tests.
 add_old_configure_assignment('PKG_CONFIG', pkg_config)
 
 include('build/moz.configure/toolchain.configure',
         when='--enable-compile-environment')
 include('build/moz.configure/memory.configure',
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -548,40 +548,52 @@ JOB_DETAILS = {
     'android-x86_64-debug': (AndroidArtifactJob, (r'public/build/target\.apk',
                                                   r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'android-aarch64-opt': (AndroidArtifactJob, (r'public/build/target\.apk',
                                                  r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'android-aarch64-debug': (AndroidArtifactJob, (r'public/build/target\.apk',
                                                    r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'linux-opt': (LinuxArtifactJob, (r'public/build/target\.tar\.bz2',
                                      r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
+    'linux-pgo': (LinuxArtifactJob, (r'public/build/target\.tar\.bz2',
+                                     r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'linux-debug': (LinuxArtifactJob, (r'public/build/target\.tar\.bz2',
                                        r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'linux64-opt': (LinuxArtifactJob, (r'public/build/target\.tar\.bz2',
                                        r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
+    'linux64-pgo': (LinuxArtifactJob, (r'public/build/target\.tar\.bz2',
+                                       r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'linux64-debug': (LinuxArtifactJob, (r'public/build/target\.tar\.bz2',
                                          r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'macosx64-opt': (MacArtifactJob, (r'public/build/firefox-(.*)\.mac\.dmg|public/build/target\.dmg',
                                       r'public/build/firefox-(.*)\.common\.tests\.(zip|tar\.gz)|'
                                       r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'macosx64-debug': (MacArtifactJob, (r'public/build/firefox-(.*)\.mac\.dmg|public/build/target\.dmg',
                                         r'public/build/firefox-(.*)\.common\.tests\.(zip|tar\.gz)|'
                                         r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'win32-opt': (WinArtifactJob, (r'public/build/firefox-(.*)\.win32\.(zip|tar\.gz)|'
                                    r'public/build/target\.(zip|tar\.gz)',
                                    r'public/build/firefox-(.*)\.common\.tests\.(zip|tar\.gz)|'
                                    r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
+    'win32-pgo': (WinArtifactJob, (r'public/build/firefox-(.*)\.win32\.(zip|tar\.gz)|'
+                                   r'public/build/target\.(zip|tar\.gz)',
+                                   r'public/build/firefox-(.*)\.common\.tests\.(zip|tar\.gz)|'
+                                   r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'win32-debug': (WinArtifactJob, (r'public/build/firefox-(.*)\.win32\.(zip|tar\.gz)|'
                                      r'public/build/target\.(zip|tar\.gz)',
                                      r'public/build/firefox-(.*)\.common\.tests\.(zip|tar\.gz)|'
                                      r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'win64-opt': (WinArtifactJob, (r'public/build/firefox-(.*)\.win64\.(zip|tar\.gz)|'
                                    r'public/build/target\.(zip|tar\.gz)',
                                    r'public/build/firefox-(.*)\.common\.tests\.(zip|tar\.gz)|'
                                    r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
+    'win64-pgo': (WinArtifactJob, (r'public/build/firefox-(.*)\.win64\.(zip|tar\.gz)|'
+                                   r'public/build/target\.(zip|tar\.gz)',
+                                   r'public/build/firefox-(.*)\.common\.tests\.(zip|tar\.gz)|'
+                                   r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
     'win64-debug': (WinArtifactJob, (r'public/build/firefox-(.*)\.win64\.(zip|tar\.gz)|public/build/target\.(zip|tar\.gz)',
                                      r'public/build/firefox-(.*)\.common\.tests\.(zip|tar\.gz)|'
                                      r'public/build/target\.common\.tests\.(zip|tar\.gz)')),
 }
 
 
 
 def get_job_details(job, log=None,
@@ -738,17 +750,17 @@ class TaskCache(CacheManager):
         tree = tree.split('/')[1] if '/' in tree else tree
 
         namespace = 'gecko.v2.{tree}.revision.{rev}.{product}.{job}'.format(
             rev=rev,
             tree=tree,
             product=artifact_job.product,
             job=job,
         )
-        self.log(logging.DEBUG, 'artifact',
+        self.log(logging.INFO, 'artifact',
                  {'namespace': namespace},
                  'Searching Taskcluster index with namespace: {namespace}')
         try:
             taskId = find_task_id(namespace)
         except KeyError:
             # Not all revisions correspond to pushes that produce the job we
             # care about; and even those that do may not have completed yet.
             raise ValueError('Task for {namespace} does not exist (yet)!'.format(namespace=namespace))
@@ -985,16 +997,18 @@ class Artifacts(object):
         if self._log:
             self._log(*args, **kwargs)
 
     def _guess_artifact_job(self):
         # Add the "-debug" suffix to the guessed artifact job name
         # if MOZ_DEBUG is enabled.
         if self._substs.get('MOZ_DEBUG'):
             target_suffix = '-debug'
+        elif self._substs.get('MOZ_PGO'):
+            target_suffix = '-pgo'
         else:
             target_suffix = '-opt'
 
         if self._substs.get('MOZ_BUILD_APP', '') == 'mobile/android':
             if self._substs['ANDROID_CPU_ARCH'] == 'x86_64':
                 return 'android-x86_64' + target_suffix
             if self._substs['ANDROID_CPU_ARCH'] == 'x86':
                 return 'android-x86' + target_suffix