Bug 1364588 - part 1 - use `mach artifact toolchain' to download clang Stylo packages; r=rillian
authorNathan Froyd <froydnj@mozilla.com>
Fri, 26 May 2017 16:07:01 -0400
changeset 409076 25c41cd44a0578ad9a70f8e39a2edb9c8be78693
parent 409075 1ca88950d83cf4658236eafc56c7195b3c9bcb85
child 409077 c37162881c6e5ccfbea82e5377ba2fa859c2c9b8
push id7391
push usermtabara@mozilla.com
push dateMon, 12 Jun 2017 13:08:53 +0000
treeherdermozilla-beta@2191d7f87e2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrillian
bugs1364588
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1364588 - part 1 - use `mach artifact toolchain' to download clang Stylo packages; r=rillian Using this command is more robust than our current method, and brings several benefits, such as smart caching of the downloaded toolchain. We change the clang package downloaded for Windows with this change, but bindgen has been updated to work well with LLVM 5.0, so there should be no problems.
python/mozboot/mozboot/base.py
python/mozboot/mozboot/linux_common.py
python/mozboot/mozboot/mozillabuild.py
python/mozboot/mozboot/osx.py
python/mozboot/mozboot/stylo.py
python/mozboot/mozboot/windows.py
--- a/python/mozboot/mozboot/base.py
+++ b/python/mozboot/mozboot/base.py
@@ -267,62 +267,35 @@ class BaseBootstrapper(object):
     def ensure_stylo_packages(self, state_dir):
         '''
         Install any necessary packages needed for Stylo development.
         '''
         raise NotImplementedError(
             '%s does not yet implement ensure_stylo_packages()'
             % __name__)
 
-    def install_tooltool_clang_package(self, state_dir,
-                                       package_filename, package_sha512sum):
-        TOOLTOOL_API = 'https://api.pub.build.mozilla.org/tooltool/sha512/'
+    def install_tooltool_clang_package(self, state_dir, manifest_file):
+        topsrcdir = os.path.join(os.path.dirname(__file__) '..', '..', '..')
+        abs_manifest_file = os.path.join(topsrcdir, manifest_file)
 
-        # XXX this is similar to the Android NDK download.  We should unify them.
-        download_path = os.path.join(state_dir, 'mozboot')
-        try:
-            os.makedirs(download_path)
-        except OSError as e:
-            if e.errno == errno.EEXIST and os.path.isdir(download_path):
-                pass
-            else:
-                raise
-
-        try:
-            package_url = TOOLTOOL_API + package_sha512sum
-            downloaded_filename = os.path.join(download_path, package_sha512sum)
-            print('Downloading clang package from', package_url)
-            self.http_download_and_save(package_url, downloaded_filename,
-                                        package_sha512sum, 'sha512')
+        mach_binary = os.path.join(topsrcdir, 'mach')
+        if not os.path.exists(mach_binary):
+            raise ValueError("mach not found at %s" % mach_binary)
 
-            # We don't have to handle a great variety of archive types here.
-            if package_filename.endswith('tar.gz'):
-                cmd = ['tar', 'zxf', downloaded_filename]
-            elif package_filename.endswith('.tar.bz2'):
-                cmd = ['tar', 'jxf', downloaded_filename]
-            elif package_filename.endswith('.tar.xz'):
-                cmd = ['tar', 'Jxf', downloaded_filename]
-            else:
-                raise NotImplementedError("Don't know how to unpack file: %s"
-                                          % package_filename)
+        # If Python can't figure out what its own executable is, there's little
+        # chance we're going to be able to execute mach on its own, particularly
+        # on Windows.
+        if not sys.executable:
+            raise ValueError("cannot determine path to Python executable")
 
-            print('Download complete!')
-            print('Unpacking %s...' % downloaded_filename)
-
-            with open(os.devnull, 'w') as stdout:
-                subprocess.check_call(cmd, stdout=stdout, cwd=state_dir)
+        cmd = [sys.executable, mach_binary, 'artifact', 'toolchain',
+               '--tooltool-manifest', abs_manifest_file,
+               'clang']
 
-            print('Unpacking %s...DONE' % downloaded_filename)
-
-        finally:
-            try:
-                os.remove(downloaded_filename)
-            except OSError as e:
-                if e.errno != errno.ENOENT:
-                    raise
+        subprocess.check_call(cmd, cwd=state_dir)
 
     def which(self, name):
         """Python implementation of which.
 
         It returns the path of an executable or None if it couldn't be found.
         """
         for path in os.environ['PATH'].split(os.pathsep):
             test = os.path.join(path, name)
--- a/python/mozboot/mozboot/linux_common.py
+++ b/python/mozboot/mozboot/linux_common.py
@@ -6,9 +6,9 @@
 # needed to install Stylo dependencies.  This class must come before
 # BaseBootstrapper in the inheritance list.
 class StyloInstall(object):
     def __init__(self, **kwargs):
         pass
 
     def ensure_stylo_packages(self, state_dir):
         import stylo
-        self.install_tooltool_clang_package(state_dir, **stylo.LINUX)
+        self.install_tooltool_clang_package(state_dir, stylo.LINUX)
--- a/python/mozboot/mozboot/mozillabuild.py
+++ b/python/mozboot/mozboot/mozillabuild.py
@@ -86,17 +86,17 @@ class MozillaBuildBootstrapper(BaseBoots
         pass
 
     def suggest_browser_mozconfig(self):
         if self.stylo:
             print(STYLO_MOZCONFIG.format(state_dir=self.state_dir))
 
     def ensure_stylo_packages(self, state_dir):
         import stylo
-        self.install_tooltool_clang_package(state_dir, **stylo.WINDOWS)
+        self.install_tooltool_clang_package(state_dir, stylo.WINDOWS)
 
     def _update_package_manager(self):
         pass
 
     def run(self, command):
         subprocess.check_call(command, stdin=sys.stdin)
 
     def pip_install(self, *packages):
--- a/python/mozboot/mozboot/osx.py
+++ b/python/mozboot/mozboot/osx.py
@@ -518,17 +518,17 @@ class OSXBootstrapper(BaseBootstrapper):
                 if path == check:
                     print(BAD_PATH_ORDER % (check, p_dir, p_dir, check, p_dir))
                     sys.exit(1)
 
         return active_name.lower()
 
     def ensure_stylo_packages(self, state_dir):
         import stylo
-        self.install_tooltool_clang_package(state_dir, **stylo.OSX)
+        self.install_tooltool_clang_package(state_dir, stylo.OSX)
 
     def install_homebrew(self):
         print(PACKAGE_MANAGER_INSTALL % ('Homebrew', 'Homebrew', 'Homebrew', 'brew'))
         bootstrap = urlopen(url=HOMEBREW_BOOTSTRAP, timeout=20).read()
         with tempfile.NamedTemporaryFile() as tf:
             tf.write(bootstrap)
             tf.flush()
 
--- a/python/mozboot/mozboot/stylo.py
+++ b/python/mozboot/mozboot/stylo.py
@@ -1,70 +1,9 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function, unicode_literals
 
-# Tooltool packages generated from the script below.
-WINDOWS = {
-    'package_filename': 'clang.tar.bz2',
-    'package_sha512sum': 'cd3ed31acefd185f441632158dde73538c62bab7ebf2a8ec630985ab345938ec522983721ddb1bead1de22d5ac1571d50a958ae002364d739f2a78c6e7244222',
-}
-
-OSX = {
-    'package_filename': 'clang.tar.bz2',
-    'package_sha512sum': '0e1a556b65d6398fa812b9ceb5ce5e2dec3eda77d4a032a818182b34fc8ce602412f42388bb1fda6bea265d35c1dde3847a730b264fec01cd7e3dcfd39941660',
-}
-
-LINUX = {
-    'package_filename': 'clang.tar.xz',
-    'package_sha512sum': '52f3fc23f0f5c98050f8b0ac7c92a6752d067582a16f712a5a58074be98975d594f9e36249fc2be7f1cc2ca6d509c663faaf2bea66f949243cc1f41651638ba6',
-}
-
-if __name__ == '__main__':
-    '''Allow invoking the module as a utility to update tooltool downloads.'''
-    import json
-    import os
-    import sys
-
-    mod_path = os.path.dirname(__file__)
-    browser_config_dir = os.path.join('browser', 'config', 'tooltool-manifests')
-
-    os_map = {
-        'WINDOWS': ('win64', 'clang.manifest'),
-        'OSX': ('macosx64', 'releng.manifest'),
-        'LINUX': ('linux64', 'releng.manifest'),
-    }
-
-    for os_name, (os_dir, f) in os_map.iteritems():
-        manifest_file = os.path.join(browser_config_dir, os_dir, f)
-        abspath = os.path.join(mod_path, '..', '..', '..', manifest_file)
-        with open(abspath, 'r') as s:
-            manifest = json.load(s)
-            entries = filter(lambda x: x['filename'].startswith('clang'), manifest)
-            if not entries:
-                print('ERROR: could not find clang tooltool entry in %s' % manifest_file)
-                sys.exit(1)
-            if len(entries) > 1:
-                print('ERROR: too many clang entries in %s' % manifest_file)
-                sys.exit(1)
-
-            clang = entries[0]
-            if clang['algorithm'] != 'sha512':
-                print("ERROR: don't know how to handle digest %s in %s" % (clang['algorithm'],
-                                                                           manifest_file))
-                sys.exit(1)
-
-            FORMAT_STRING = """{os} = {{
-    'package_filename': '{filename}',
-    'package_sha512sum': '{digest}',
-}}
-"""
-            digest = clang['digest']
-            if os_name == 'WINDOWS':
-                # The only clang version we can retrieve from the tooltool manifest
-                # doesn't work with Stylo bindgen due to regressions in LLVM.  This
-                # is an older version that works.
-                digest = 'cd3ed31acefd185f441632158dde73538c62bab7ebf2a8ec630985ab345938ec522983721ddb1bead1de22d5ac1571d50a958ae002364d739f2a78c6e7244222'
-            print(FORMAT_STRING.format(os=os_name,
-                                       filename=clang['filename'],
-                                       digest=digest))
+WINDOWS = 'browser/config/tooltool-manifests/win64/clang.manifest'
+OSX = 'browser/config/tooltool-manifests/macosx64/releng.manifest'
+LINUX = 'browser/config/tooltool-manifests/linux64/clang.manifest'
--- a/python/mozboot/mozboot/windows.py
+++ b/python/mozboot/mozboot/windows.py
@@ -81,17 +81,17 @@ class WindowsBootstrapper(BaseBootstrapp
         raise NotImplementedError('We do not support building Android on Windows. Sorry!')
 
     def suggest_browser_mozconfig(self):
         if self.stylo:
             print(STYLO_MOZCONFIG.format(state_dir=self.state_dir))
 
     def ensure_stylo_packages(self, state_dir):
         import stylo
-        self.install_tooltool_clang_package(state_dir, **stylo.WINDOWS)
+        self.install_tooltool_clang_package(state_dir, stylo.WINDOWS)
 
     def _update_package_manager(self):
         self.pacman_update()
 
     def run(self, command):
         subprocess.check_call(command, stdin=sys.stdin)
 
     def pacman_update(self):