Bug 1509296 - Download clang based static-analysis during bootstrap. r=nalexander
authorAndi-Bogdan Postelnicu <bpostelnicu@mozilla.com>
Tue, 27 Nov 2018 19:43:37 +0000
changeset 507581 d44189ff5da2f3e8d2a0c24bc81d04523e616aa0
parent 507580 cc540ed50749d32dfc41981b568de1abdd8b816b
child 507582 d40fbd0c1a779633e1bd1f0c77cfcbb4f9e6e35a
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1509296
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1509296 - Download clang based static-analysis during bootstrap. r=nalexander Differential Revision: https://phabricator.services.mozilla.com/D12658
python/mozboot/mozboot/archlinux.py
python/mozboot/mozboot/base.py
python/mozboot/mozboot/bootstrap.py
python/mozboot/mozboot/centosfedora.py
python/mozboot/mozboot/debian.py
python/mozboot/mozboot/freebsd.py
python/mozboot/mozboot/gentoo.py
python/mozboot/mozboot/linux_common.py
python/mozboot/mozboot/mozillabuild.py
python/mozboot/mozboot/openbsd.py
python/mozboot/mozboot/osx.py
python/mozboot/mozboot/windows.py
--- a/python/mozboot/mozboot/archlinux.py
+++ b/python/mozboot/mozboot/archlinux.py
@@ -6,20 +6,21 @@ from __future__ import absolute_import, 
 
 import os
 import sys
 import tempfile
 import subprocess
 import glob
 
 from mozboot.base import BaseBootstrapper
-from mozboot.linux_common import NodeInstall, StyloInstall
+from mozboot.linux_common import NodeInstall, StyloInstall, ClangStaticAnalysisInstall
 
 
-class ArchlinuxBootstrapper(NodeInstall, StyloInstall, BaseBootstrapper):
+class ArchlinuxBootstrapper(NodeInstall, StyloInstall,
+                            ClangStaticAnalysisInstall, BaseBootstrapper):
     '''Archlinux experimental bootstrapper.'''
 
     SYSTEM_PACKAGES = [
         'autoconf2.13',
         'base-devel',
         'nodejs',
         'python2',
         'python2-setuptools',
--- a/python/mozboot/mozboot/base.py
+++ b/python/mozboot/mozboot/base.py
@@ -248,31 +248,57 @@ class BaseBootstrapper(object):
 
         Firefox for Android Artifact Mode needs an application and an ABI set,
         and it needs paths to the Android SDK.
         '''
         raise NotImplementedError(
             '%s does not yet implement suggest_mobile_android_artifact_mode_mozconfig()'
             % __name__)
 
+    def ensure_clang_static_analysis_package(self, checkout_root):
+        '''
+        Install the clang static analysis package
+        '''
+        raise NotImplementedError(
+            '%s does not yet implement ensure_clang_static_analysis_package()'
+            % __name__)
+
     def ensure_stylo_packages(self, state_dir, checkout_root):
         '''
         Install any necessary packages needed for Stylo development.
         '''
         raise NotImplementedError(
             '%s does not yet implement ensure_stylo_packages()'
             % __name__)
 
     def ensure_node_packages(self, state_dir, checkout_root):
         '''
         Install any necessary packages needed to supply NodeJS'''
         raise NotImplementedError(
             '%s does not yet implement ensure_node_packages()'
             % __name__)
 
+    def install_toolchain_static_analysis(self, checkout_root):
+        mach_binary = os.path.join(checkout_root, 'mach')
+        mach_binary = os.path.abspath(mach_binary)
+        if not os.path.exists(mach_binary):
+            raise ValueError("mach not found at %s" % mach_binary)
+
+        if not sys.executable:
+            raise ValueError("cannot determine path to Python executable")
+
+        cmd = [sys.executable, mach_binary, 'static-analysis', 'install',
+               '--force', '--minimal-install']
+
+        from subprocess import CalledProcessError
+        try:
+            subprocess.check_call(cmd)
+        except CalledProcessError as e:
+            print(e.output)
+
     def install_toolchain_artifact(self, state_dir, checkout_root, toolchain_job):
         mach_binary = os.path.join(checkout_root, 'mach')
         mach_binary = os.path.abspath(mach_binary)
         if not os.path.exists(mach_binary):
             raise ValueError("mach not found at %s" % mach_binary)
 
         # If Python can't figure out what its own executable is, there's little
         # chance we're going to be able to execute mach on its own, particularly
--- a/python/mozboot/mozboot/bootstrap.py
+++ b/python/mozboot/mozboot/bootstrap.py
@@ -363,28 +363,29 @@ class Bootstrapper(object):
         return state_dir_available, state_dir
 
     def maybe_install_private_packages_or_exit(self, state_dir,
                                                state_dir_available,
                                                have_clone,
                                                checkout_root):
         # Install the clang packages needed for building the style system, as
         # well as the version of NodeJS that we currently support.
-
+        # Also install the clang static-analysis package by default
         # The best place to install our packages is in the state directory
         # we have.  We should have created one above in non-interactive mode.
         if not state_dir_available:
             print(STYLO_NODEJS_DIRECTORY_MESSAGE.format(statedir=state_dir))
             sys.exit(1)
 
         if not have_clone:
             print(STYLE_NODEJS_REQUIRES_CLONE)
             sys.exit(1)
 
         self.instance.state_dir = state_dir
+        self.instance.ensure_clang_static_analysis_package(checkout_root)
         self.instance.ensure_stylo_packages(state_dir, checkout_root)
         self.instance.ensure_node_packages(state_dir, checkout_root)
 
     def check_telemetry_opt_in(self, state_dir):
         # We can't prompt the user.
         if self.instance.no_interactive:
             return
         # Don't prompt if the user already has a setting for this value.
--- a/python/mozboot/mozboot/centosfedora.py
+++ b/python/mozboot/mozboot/centosfedora.py
@@ -2,20 +2,21 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 import platform
 
 from mozboot.base import BaseBootstrapper
-from mozboot.linux_common import NodeInstall, StyloInstall
+from mozboot.linux_common import NodeInstall, StyloInstall, ClangStaticAnalysisInstall
 
 
-class CentOSFedoraBootstrapper(NodeInstall, StyloInstall, BaseBootstrapper):
+class CentOSFedoraBootstrapper(NodeInstall, StyloInstall,
+                               ClangStaticAnalysisInstall, BaseBootstrapper):
     def __init__(self, distro, version, dist_id, **kwargs):
         BaseBootstrapper.__init__(self, **kwargs)
 
         self.distro = distro
         self.version = version
         self.dist_id = dist_id
 
         self.group_packages = []
--- a/python/mozboot/mozboot/debian.py
+++ b/python/mozboot/mozboot/debian.py
@@ -1,16 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function
 
 from mozboot.base import BaseBootstrapper
-from mozboot.linux_common import NodeInstall, StyloInstall
+from mozboot.linux_common import NodeInstall, StyloInstall, ClangStaticAnalysisInstall
 
 
 MERCURIAL_INSTALL_PROMPT = '''
 Mercurial releases a new version every 3 months and your distro's package
 may become out of date. This may cause incompatibility with some
 Mercurial extensions that rely on new Mercurial features. As a result,
 you may not have an optimal version control experience.
 
@@ -23,17 +23,18 @@ How would you like to continue?
 1) Install a modern Mercurial via pip (recommended)
 2) Install a legacy Mercurial via apt
 3) Do not install Mercurial
 
 Choice:
 '''.strip()
 
 
-class DebianBootstrapper(NodeInstall, StyloInstall, BaseBootstrapper):
+class DebianBootstrapper(NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
+                         BaseBootstrapper):
     # These are common packages for all Debian-derived distros (such as
     # Ubuntu).
     COMMON_PACKAGES = [
         'autoconf2.13',
         'build-essential',
         'nodejs',
         'python-dev',
         'python-pip',
--- a/python/mozboot/mozboot/freebsd.py
+++ b/python/mozboot/mozboot/freebsd.py
@@ -61,16 +61,20 @@ class FreeBSDBootstrapper(BaseBootstrapp
 
     def install_browser_artifact_mode_packages(self):
         self.ensure_browser_packages(artifact_mode=True)
 
     def ensure_browser_packages(self, artifact_mode=False):
         # TODO: Figure out what not to install for artifact mode
         self.pkg_install(*self.browser_packages)
 
+    def ensure_clang_static_analysis_package(self, checkout_root):
+        # TODO: we don't ship clang base static analysis for this platform
+        pass
+
     def ensure_stylo_packages(self, state_dir, checkout_root):
         # Clang / llvm already installed as browser package
         self.pkg_install('rust-cbindgen')
 
     def ensure_node_packages(self, state_dir, checkout_root):
         self.pkg_install('npm')
 
     def upgrade_mercurial(self, current):
--- a/python/mozboot/mozboot/gentoo.py
+++ b/python/mozboot/mozboot/gentoo.py
@@ -1,27 +1,28 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 from mozboot.base import BaseBootstrapper
-from mozboot.linux_common import NodeInstall, StyloInstall
+from mozboot.linux_common import NodeInstall, StyloInstall, ClangStaticAnalysisInstall
 
 try:
     from urllib2 import urlopen
 except ImportError:
     from urllib.request import urlopen
 
 import re
 import subprocess
 
 
-class GentooBootstrapper(NodeInstall, StyloInstall, BaseBootstrapper):
+class GentooBootstrapper(NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
+                         BaseBootstrapper):
     def __init__(self, version, dist_id, **kwargs):
         BaseBootstrapper.__init__(self, **kwargs)
 
         self.version = version
         self.dist_id = dist_id
 
     def install_system_packages(self):
         self.run_as_root(['emerge', '--noreplace', '--quiet', 'nodejs'])
--- a/python/mozboot/mozboot/linux_common.py
+++ b/python/mozboot/mozboot/linux_common.py
@@ -21,8 +21,16 @@ class StyloInstall(object):
 
 class NodeInstall(object):
     def __init__(self, **kwargs):
         pass
 
     def ensure_node_packages(self, state_dir, checkout_root):
         from mozboot import node
         self.install_toolchain_artifact(state_dir, checkout_root, node.LINUX)
+
+
+class ClangStaticAnalysisInstall(object):
+    def __init__(self, **kwargs):
+        pass
+
+    def ensure_clang_static_analysis_package(self, checkout_root):
+        self.install_toolchain_static_analysis(checkout_root)
--- a/python/mozboot/mozboot/mozillabuild.py
+++ b/python/mozboot/mozboot/mozillabuild.py
@@ -40,16 +40,19 @@ class MozillaBuildBootstrapper(BaseBoots
         pass
 
     def install_mobile_android_packages(self):
         pass
 
     def install_mobile_android_artifact_mode_packages(self):
         pass
 
+    def ensure_clang_static_analysis_package(self, checkout_root):
+        self.install_toolchain_static_analysis(checkout_root)
+
     def ensure_stylo_packages(self, state_dir, checkout_root):
         from mozboot import stylo
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.WINDOWS_CLANG)
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.WINDOWS_CBINDGEN)
 
     def ensure_node_packages(self, state_dir, checkout_root):
         from mozboot import node
         self.install_toolchain_artifact(
--- a/python/mozboot/mozboot/openbsd.py
+++ b/python/mozboot/mozboot/openbsd.py
@@ -42,14 +42,18 @@ class OpenBSDBootstrapper(BaseBootstrapp
     def install_browser_artifact_mode_packages(self):
         self.ensure_browser_packages(artifact_mode=True)
 
     def ensure_browser_packages(self, artifact_mode=False):
         # TODO: Figure out what not to install for artifact mode
         # we use -z because there's no other way to say "any autoconf-2.13"
         self.run_as_root(['pkg_add', '-z'] + self.browser_packages)
 
+    def ensure_clang_static_analysis_package(self, checkout_root):
+        # TODO: we don't ship clang base static analysis for this platform
+        pass
+
     def ensure_stylo_packages(self, state_dir, checkout_root):
         # Clang / llvm already installed as browser package
         self.run_as_root(['pkg_add', 'cbindgen'])
 
     def ensure_node_packages(self, state_dir, checkout_root):
         self.run_as_root(['pkg_add', 'node'])
--- a/python/mozboot/mozboot/osx.py
+++ b/python/mozboot/mozboot/osx.py
@@ -499,16 +499,19 @@ class OSXBootstrapper(BaseBootstrapper):
 
             for check in ('/bin', '/usr/bin'):
                 if path == check:
                     print(BAD_PATH_ORDER % (check, p_dir, p_dir, check, p_dir))
                     sys.exit(1)
 
         return active_name.lower()
 
+    def ensure_clang_static_analysis_package(self, checkout_root):
+        self.install_toolchain_static_analysis(checkout_root)
+
     def ensure_stylo_packages(self, state_dir, checkout_root):
         from mozboot import stylo
         # We installed clang via homebrew earlier.
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.MACOS_CBINDGEN)
 
     def ensure_node_packages(self, state_dir, checkout_root):
         # XXX from necessary?
         from mozboot import node
--- a/python/mozboot/mozboot/windows.py
+++ b/python/mozboot/mozboot/windows.py
@@ -67,16 +67,19 @@ class WindowsBootstrapper(BaseBootstrapp
         self.pacman_install(*self.BROWSER_PACKAGES)
 
     def install_mobile_android_packages(self):
         raise NotImplementedError('We do not support building Android on Windows. Sorry!')
 
     def install_mobile_android_artifact_mode_packages(self):
         raise NotImplementedError('We do not support building Android on Windows. Sorry!')
 
+    def ensure_clang_static_analysis_package(self, checkout_root):
+        self.install_toolchain_static_analysis(checkout_root)
+
     def ensure_stylo_packages(self, state_dir, checkout_root):
         from mozboot import stylo
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.WINDOWS_CLANG)
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.WINDOWS_CBINDGEN)
 
     def ensure_node_packages(self, state_dir, checkout_root):
         from mozboot import node
         self.install_toolchain_artifact(