Bug 1529195 - Allow to opt-in to symbols/host-bin artifacts from the CLI. r=chmanchester
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 21 Feb 2019 22:53:43 +0000
changeset 460453 b78a6ad317cfc135f6632d9e4cacecc34e4fa334
parent 460452 e953a0b9aefc7d986a99f44a2b551484ca0d8844
child 460454 4e79e369f1ee46395375a92400b877127a4f6cea
push id35593
push userccoroiu@mozilla.com
push dateFri, 22 Feb 2019 16:25:14 +0000
treeherdermozilla-central@a1d118e856dd [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1529195
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1529195 - Allow to opt-in to symbols/host-bin artifacts from the CLI. r=chmanchester and to opt-out of test artifacts. Depends on D20445 Differential Revision: https://phabricator.services.mozilla.com/D20446
Makefile.in
python/mozbuild/mozbuild/artifacts.py
python/mozbuild/mozbuild/mach_commands.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -158,18 +158,22 @@ binaries::
 endif
 endif
 else # !MOZ_PROFILE_USE (normal build)
 recurse_pre-export:: install-manifests
 binaries::
 	@$(MAKE) install-manifests install_manifests=dist/include
 endif
 
+# Host binaries are not produced for macOS consumers: that is, there's
+# no macOS-hosted job to produce them at this time.  Therefore we
+# enable --host-bins only for automation builds, which only require Linux and
+# Windows host binaries.
 recurse_artifact:
-	$(topsrcdir)/mach --log-no-times artifact install
+	$(topsrcdir)/mach --log-no-times artifact install$(if $(MOZ_ARTIFACT_BUILD_SYMBOLS), --symbols)$(if $(MOZ_AUTOMATION), --host-bins)
 
 ifdef MOZ_WIDGET_TOOLKIT
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites
 include $(topsrcdir)/testing/testsuite-targets.mk
 endif
 endif
 
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -143,21 +143,24 @@ class ArtifactJob(object):
     }
 
     # We can tell our input is a test archive by this suffix, which happens to
     # be the same across platforms.
     _test_zip_archive_suffix = '.common.tests.zip'
     _test_tar_archive_suffix = '.common.tests.tar.gz'
 
     def __init__(self, log=None,
+                 download_tests=True,
                  download_symbols=False,
                  download_host_bins=False,
                  substs=None):
         self._package_re = re.compile(self.package_re)
-        self._tests_re = re.compile(r'public/build/target\.common\.tests\.(zip|tar\.gz)')
+        self._tests_re = None
+        if download_tests:
+            self._tests_re = re.compile(r'public/build/target\.common\.tests\.(zip|tar\.gz)')
         self._host_bins_re = None
         if download_host_bins:
             self._host_bins_re = re.compile(r'public/build/host/bin/(mar|mbsdiff)(.exe)?')
         self._log = log
         self._substs = substs
         self._symbols_archive_suffix = None
         if download_symbols:
             self._symbols_archive_suffix = 'crashreporter-symbols.zip'
@@ -894,42 +897,38 @@ class ArtifactCache(object):
         self._persist_limit.remove_all()
 
 
 class Artifacts(object):
     '''Maintain state to efficiently fetch build artifacts from a Firefox tree.'''
 
     def __init__(self, tree, substs, defines, job=None, log=None,
                  cache_dir='.', hg=None, git=None, skip_cache=False,
-                 topsrcdir=None):
+                 topsrcdir=None, download_tests=True, download_symbols=False,
+                 download_host_bins=False):
         if (hg and git) or (not hg and not git):
             raise ValueError("Must provide path to exactly one of hg and git")
 
         self._substs = substs
-        self._download_symbols = self._substs.get('MOZ_ARTIFACT_BUILD_SYMBOLS', False)
-        # Host binaries are not produced for macOS consumers: that is, there's
-        # no macOS-hosted job to produce them at this time.  Therefore we
-        # enable this only for automation builds, which only require Linux and
-        # Windows host binaries.
-        self._download_host_bins = self._substs.get('MOZ_AUTOMATION', False)
         self._defines = defines
         self._tree = tree
         self._job = job or self._guess_artifact_job()
         self._log = log
         self._hg = hg
         self._git = git
         self._cache_dir = cache_dir
         self._skip_cache = skip_cache
         self._topsrcdir = topsrcdir
 
         try:
             cls = JOB_DETAILS[self._job]
             self._artifact_job = cls(log=self._log,
-                                     download_symbols=self._download_symbols,
-                                     download_host_bins=self._download_host_bins,
+                                     download_tests=download_tests,
+                                     download_symbols=download_symbols,
+                                     download_host_bins=download_host_bins,
                                      substs=self._substs)
         except KeyError:
             self.log(logging.INFO, 'artifact',
                 {'job': self._job},
                 'Unknown job {job}')
             raise KeyError("Unknown job")
 
         self._task_cache = TaskCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache)
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1253,17 +1253,19 @@ class PackageFrontend(MachCommandBase):
         object directory.  Use |mach artifact last| to see what binary artifacts
         were last used.
 
         Never build libxul again!
 
         '''
         pass
 
-    def _make_artifacts(self, tree=None, job=None, skip_cache=False):
+    def _make_artifacts(self, tree=None, job=None, skip_cache=False,
+                        download_tests=True, download_symbols=False,
+                        download_host_bins=False):
         state_dir = self._mach_context.state_dir
         cache_dir = os.path.join(state_dir, 'package-frontend')
 
         hg = None
         if conditions.is_hg(self):
             hg = self.substs['HG']
 
         git = None
@@ -1274,32 +1276,42 @@ class PackageFrontend(MachCommandBase):
         # We can't derive JOB_CHOICES from JOB_DETAILS because we don't want to
         # import the artifacts module globally ; and this module can't be
         # imported in unit tests, so do the check here.
         assert set(JOB_DETAILS.keys()) == JOB_CHOICES
 
         artifacts = Artifacts(tree, self.substs, self.defines, job,
                               log=self.log, cache_dir=cache_dir,
                               skip_cache=skip_cache, hg=hg, git=git,
-                              topsrcdir=self.topsrcdir)
+                              topsrcdir=self.topsrcdir,
+                              download_tests=download_tests,
+                              download_symbols=download_symbols,
+                              download_host_bins=download_host_bins)
         return artifacts
 
     @ArtifactSubCommand('artifact', 'install',
         'Install a good pre-built artifact.')
     @CommandArgument('source', metavar='SRC', nargs='?', type=str,
         help='Where to fetch and install artifacts from.  Can be omitted, in '
             'which case the current hg repository is inspected; an hg revision; '
             'a remote URL; or a local file.',
         default=None)
     @CommandArgument('--skip-cache', action='store_true',
         help='Skip all local caches to force re-fetching remote artifacts.',
         default=False)
-    def artifact_install(self, source=None, skip_cache=False, tree=None, job=None, verbose=False):
+    @CommandArgument('--no-tests', action='store_true', help="Don't install tests.")
+    @CommandArgument('--symbols', action='store_true', help='Download symbols.')
+    @CommandArgument('--host-bins', action='store_true', help='Download host binaries.')
+    def artifact_install(self, source=None, skip_cache=False, tree=None, job=None, verbose=False,
+                         no_tests=False, symbols=False, host_bins=False):
         self._set_log_level(verbose)
-        artifacts = self._make_artifacts(tree=tree, job=job, skip_cache=skip_cache)
+        artifacts = self._make_artifacts(tree=tree, job=job, skip_cache=skip_cache,
+                                         download_tests=not no_tests,
+                                         download_symbols=symbols,
+                                         download_host_bins=host_bins)
 
         return artifacts.install_from(source, self.distdir)
 
     @ArtifactSubCommand('artifact', 'clear-cache',
         'Delete local artifacts and reset local artifact cache.')
     def artifact_clear_cache(self, tree=None, job=None, verbose=False):
         self._set_log_level(verbose)
         artifacts = self._make_artifacts(tree=tree, job=job)