Bug 1521186 - Download nasm toolchain on Windows and Linux. r=firefox-build-system-reviewers,mshal
authorThomas Daede <tdaede@mozilla.com>
Mon, 11 Mar 2019 18:00:13 +0000
changeset 521436 de512e13803ba10b6354874ff9d9081f48c7b5fa
parent 521435 403a207f3e164beac577f4f156df218301b02c28
child 521437 aa539ea8ce2790a991b512b1aea46ab260d9c0f5
push id10866
push usernerli@mozilla.com
push dateTue, 12 Mar 2019 18:59:09 +0000
treeherdermozilla-beta@445c24a51727 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfirefox-build-system-reviewers, mshal
bugs1521186
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1521186 - Download nasm toolchain on Windows and Linux. r=firefox-build-system-reviewers,mshal Differential Revision: https://phabricator.services.mozilla.com/D22798
build/moz.configure/toolchain.configure
python/mozboot/mozboot/archlinux.py
python/mozboot/mozboot/base.py
python/mozboot/mozboot/bootstrap.py
python/mozboot/mozboot/centosfedora.py
python/mozboot/mozboot/debian.py
python/mozboot/mozboot/freebsd.py
python/mozboot/mozboot/gentoo.py
python/mozboot/mozboot/linux_common.py
python/mozboot/mozboot/mozillabuild.py
python/mozboot/mozboot/nasm.py
python/mozboot/mozboot/openbsd.py
python/mozboot/mozboot/osx.py
python/mozboot/mozboot/windows.py
--- a/build/moz.configure/toolchain.configure
+++ b/build/moz.configure/toolchain.configure
@@ -86,85 +86,16 @@ def yasm_asflags(yasm, target):
         if asflags:
             asflags += ['-rnasm', '-pnasm']
         return asflags
 
 
 set_config('YASM_ASFLAGS', yasm_asflags)
 
 
-# nasm detection
-# ==============================================================
-nasm = check_prog('NASM', ['nasm'], allow_missing=True)
-
-
-@depends_if(nasm)
-@checking('nasm version')
-def nasm_version(nasm):
-    (retcode, stdout, _) = get_cmd_output(nasm, '-v')
-    if retcode:
-        # mac stub binary
-        return None
-
-    version = stdout.splitlines()[0].split()[2]
-    return Version(version)
-
-
-@depends_if(nasm_version)
-def nasm_major_version(nasm_version):
-    return str(nasm_version.major)
-
-
-@depends_if(nasm_version)
-def nasm_minor_version(nasm_version):
-    return str(nasm_version.minor)
-
-
-set_config('NASM_MAJOR_VERSION', nasm_major_version)
-set_config('NASM_MINOR_VERSION', nasm_minor_version)
-
-
-@depends(nasm, target)
-def nasm_asflags(nasm, target):
-    if nasm:
-        asflags = {
-            ('OSX', 'x86'): ['-f', 'macho32'],
-            ('OSX', 'x86_64'): ['-f', 'macho64'],
-            ('WINNT', 'x86'): ['-f', 'win32'],
-            ('WINNT', 'x86_64'): ['-f', 'win64'],
-        }.get((target.os, target.cpu), None)
-        if asflags is None:
-            # We're assuming every x86 platform we support that's
-            # not Windows or Mac is ELF.
-            if target.cpu == 'x86':
-                asflags = ['-f', 'elf32']
-            elif target.cpu == 'x86_64':
-                asflags = ['-f', 'elf64']
-        return asflags
-
-
-set_config('NASM_ASFLAGS', nasm_asflags)
-
-@depends(nasm_asflags)
-def have_nasm(value):
-    if value:
-        return True
-
-
-@depends(yasm_asflags)
-def have_yasm(yasm_asflags):
-    if yasm_asflags:
-        return True
-
-set_config('HAVE_NASM', have_nasm)
-
-set_config('HAVE_YASM', have_yasm)
-# Until the YASM variable is not necessary in old-configure.
-add_old_configure_assignment('YASM', have_yasm)
-
 # Android NDK
 # ==============================================================
 
 
 @depends('--disable-compile-environment', target)
 def compiling_android(compile_env, target):
     return compile_env and target.os == 'Android'
 
@@ -788,16 +719,19 @@ def toolchain_search_path_for(host_or_ta
         mozbuild_state_dir = environ.get('MOZBUILD_STATE_PATH',
                                          os.path.expanduser(os.path.join('~', '.mozbuild')))
         bootstrap_clang_path = os.path.join(mozbuild_state_dir, 'clang', 'bin')
         result.append(bootstrap_clang_path)
 
         bootstrap_cbindgen_path = os.path.join(mozbuild_state_dir, 'cbindgen')
         result.append(bootstrap_cbindgen_path)
 
+        bootstrap_nasm_path = os.path.join(mozbuild_state_dir, 'nasm')
+        result.append(bootstrap_nasm_path)
+
         # Also add the rustup install directory for cargo/rustc.
         rustup_path = os.path.expanduser(os.path.join('~', '.cargo', 'bin'))
         result.append(rustup_path)
 
         return result
     return toolchain_search_path
 
 
@@ -1724,16 +1658,85 @@ def security_hardening_cflags(hardening_
     )
 
 
 add_old_configure_assignment('MOZ_HARDENING_CFLAGS', security_hardening_cflags.flags)
 add_old_configure_assignment('MOZ_HARDENING_LDFLAGS', security_hardening_cflags.ldflags)
 add_old_configure_assignment('MOZ_HARDENING_CFLAGS_JS', security_hardening_cflags.js_flags)
 add_old_configure_assignment('MOZ_HARDENING_LDFLAGS_JS', security_hardening_cflags.js_ldflags)
 
+# nasm detection
+# ==============================================================
+nasm = check_prog('NASM', ['nasm'], allow_missing=True, paths=toolchain_search_path)
+
+
+@depends_if(nasm)
+@checking('nasm version')
+def nasm_version(nasm):
+    (retcode, stdout, _) = get_cmd_output(nasm, '-v')
+    if retcode:
+        # mac stub binary
+        return None
+
+    version = stdout.splitlines()[0].split()[2]
+    return Version(version)
+
+
+@depends_if(nasm_version)
+def nasm_major_version(nasm_version):
+    return str(nasm_version.major)
+
+
+@depends_if(nasm_version)
+def nasm_minor_version(nasm_version):
+    return str(nasm_version.minor)
+
+
+set_config('NASM_MAJOR_VERSION', nasm_major_version)
+set_config('NASM_MINOR_VERSION', nasm_minor_version)
+
+
+@depends(nasm, target)
+def nasm_asflags(nasm, target):
+    if nasm:
+        asflags = {
+            ('OSX', 'x86'): ['-f', 'macho32'],
+            ('OSX', 'x86_64'): ['-f', 'macho64'],
+            ('WINNT', 'x86'): ['-f', 'win32'],
+            ('WINNT', 'x86_64'): ['-f', 'win64'],
+        }.get((target.os, target.cpu), None)
+        if asflags is None:
+            # We're assuming every x86 platform we support that's
+            # not Windows or Mac is ELF.
+            if target.cpu == 'x86':
+                asflags = ['-f', 'elf32']
+            elif target.cpu == 'x86_64':
+                asflags = ['-f', 'elf64']
+        return asflags
+
+
+set_config('NASM_ASFLAGS', nasm_asflags)
+
+@depends(nasm_asflags)
+def have_nasm(value):
+    if value:
+        return True
+
+
+@depends(yasm_asflags)
+def have_yasm(yasm_asflags):
+    if yasm_asflags:
+        return True
+
+set_config('HAVE_NASM', have_nasm)
+
+set_config('HAVE_YASM', have_yasm)
+# Until the YASM variable is not necessary in old-configure.
+add_old_configure_assignment('YASM', have_yasm)
+
 # Code Coverage
 # ==============================================================
 
 js_option('--enable-coverage', env='MOZ_CODE_COVERAGE',
           help='Enable code coverage')
 
 
 @depends('--enable-coverage')
--- a/python/mozboot/mozboot/archlinux.py
+++ b/python/mozboot/mozboot/archlinux.py
@@ -94,16 +94,20 @@ class ArchlinuxBootstrapper(NodeInstall,
     def install_mobile_android_artifact_mode_packages(self):
         self.ensure_mobile_android_packages(artifact_mode=True)
 
     def ensure_browser_packages(self, artifact_mode=False):
         # TODO: Figure out what not to install for artifact mode
         self.aur_install(*self.BROWSER_AUR_PACKAGES)
         self.pacman_install(*self.BROWSER_PACKAGES)
 
+    def ensure_nasm_packages(self, state_dir, checkout_root):
+        # installed via ensure_browser_packages
+        pass
+
     def ensure_mobile_android_packages(self, artifact_mode=False):
         # Multi-part process:
         # 1. System packages.
         # 2. Android SDK. Android NDK only if we are not in artifact mode. Android packages.
 
         # 1. This is hard to believe, but the Android SDK binaries are 32-bit
         # and that conflicts with 64-bit Arch installations out of the box.  The
         # solution is to add the multilibs repository; unfortunately, this
--- a/python/mozboot/mozboot/base.py
+++ b/python/mozboot/mozboot/base.py
@@ -267,16 +267,24 @@ class BaseBootstrapper(object):
     def ensure_stylo_packages(self, state_dir, checkout_root):
         '''
         Install any necessary packages needed for Stylo development.
         '''
         raise NotImplementedError(
             '%s does not yet implement ensure_stylo_packages()'
             % __name__)
 
+    def ensure_nasm_packages(self, state_dir, checkout_root):
+        '''
+        Install nasm.
+        '''
+        raise NotImplementedError(
+            '%s does not yet implement ensure_nasm_packages()'
+            % __name__)
+
     def ensure_node_packages(self, state_dir, checkout_root):
         '''
         Install any necessary packages needed to supply NodeJS'''
         raise NotImplementedError(
             '%s does not yet implement ensure_node_packages()'
             % __name__)
 
     def install_toolchain_static_analysis(self, state_dir, checkout_root, toolchain_job):
--- a/python/mozboot/mozboot/bootstrap.py
+++ b/python/mozboot/mozboot/bootstrap.py
@@ -380,16 +380,17 @@ class Bootstrapper(object):
             print(STYLE_NODEJS_REQUIRES_CLONE)
             sys.exit(1)
 
         self.instance.state_dir = state_dir
         self.instance.ensure_node_packages(state_dir, checkout_root)
         if not self.instance.artifact_mode:
             self.instance.ensure_stylo_packages(state_dir, checkout_root)
             self.instance.ensure_clang_static_analysis_package(state_dir, checkout_root)
+            self.instance.ensure_nasm_packages(state_dir, checkout_root)
 
     def check_telemetry_opt_in(self, state_dir):
         # We can't prompt the user.
         if self.instance.no_interactive:
             return
         # Don't prompt if the user already has a setting for this value.
         if self.mach_context is not None and 'telemetry' in self.mach_context.settings.build:
             return
--- a/python/mozboot/mozboot/centosfedora.py
+++ b/python/mozboot/mozboot/centosfedora.py
@@ -2,20 +2,20 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 import platform
 
 from mozboot.base import BaseBootstrapper
-from mozboot.linux_common import NodeInstall, StyloInstall, ClangStaticAnalysisInstall
+from mozboot.linux_common import NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall
 
 
-class CentOSFedoraBootstrapper(NodeInstall, StyloInstall,
+class CentOSFedoraBootstrapper(NasmInstall, NodeInstall, StyloInstall,
                                ClangStaticAnalysisInstall, BaseBootstrapper):
     def __init__(self, distro, version, dist_id, **kwargs):
         BaseBootstrapper.__init__(self, **kwargs)
 
         self.distro = distro
         self.version = version
         self.dist_id = dist_id
 
--- a/python/mozboot/mozboot/debian.py
+++ b/python/mozboot/mozboot/debian.py
@@ -1,16 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function
 
 from mozboot.base import BaseBootstrapper
-from mozboot.linux_common import NodeInstall, StyloInstall, ClangStaticAnalysisInstall
+from mozboot.linux_common import NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall
 
 
 MERCURIAL_INSTALL_PROMPT = '''
 Mercurial releases a new version every 3 months and your distro's package
 may become out of date. This may cause incompatibility with some
 Mercurial extensions that rely on new Mercurial features. As a result,
 you may not have an optimal version control experience.
 
@@ -23,17 +23,17 @@ How would you like to continue?
 1) Install a modern Mercurial via pip (recommended)
 2) Install a legacy Mercurial via apt
 3) Do not install Mercurial
 
 Choice:
 '''.strip()
 
 
-class DebianBootstrapper(NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
+class DebianBootstrapper(NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
                          BaseBootstrapper):
     # These are common packages for all Debian-derived distros (such as
     # Ubuntu).
     COMMON_PACKAGES = [
         'autoconf2.13',
         'build-essential',
         'nodejs',
         'python-dev',
--- a/python/mozboot/mozboot/freebsd.py
+++ b/python/mozboot/mozboot/freebsd.py
@@ -69,13 +69,17 @@ class FreeBSDBootstrapper(BaseBootstrapp
     def ensure_clang_static_analysis_package(self, state_dir, checkout_root):
         # TODO: we don't ship clang base static analysis for this platform
         pass
 
     def ensure_stylo_packages(self, state_dir, checkout_root):
         # Clang / llvm already installed as browser package
         self.pkg_install('rust-cbindgen')
 
+    def ensure_nasm_packages(self, state_dir, checkout_root):
+        # installed via ensure_browser_packages
+        pass
+
     def ensure_node_packages(self, state_dir, checkout_root):
         self.pkg_install('npm')
 
     def upgrade_mercurial(self, current):
         self.pkg_install('mercurial')
--- a/python/mozboot/mozboot/gentoo.py
+++ b/python/mozboot/mozboot/gentoo.py
@@ -1,27 +1,27 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 from mozboot.base import BaseBootstrapper
-from mozboot.linux_common import NodeInstall, StyloInstall, ClangStaticAnalysisInstall
+from mozboot.linux_common import NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall
 
 try:
     from urllib2 import urlopen
 except ImportError:
     from urllib.request import urlopen
 
 import re
 import subprocess
 
 
-class GentooBootstrapper(NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
+class GentooBootstrapper(NasmInstall, NodeInstall, StyloInstall, ClangStaticAnalysisInstall,
                          BaseBootstrapper):
     def __init__(self, version, dist_id, **kwargs):
         BaseBootstrapper.__init__(self, **kwargs)
 
         self.version = version
         self.dist_id = dist_id
 
     def install_system_packages(self):
--- a/python/mozboot/mozboot/linux_common.py
+++ b/python/mozboot/mozboot/linux_common.py
@@ -26,16 +26,30 @@ class StyloInstall(object):
             print('Cannot install bindgen clang and cbindgen packages from taskcluster.\n'
                   'Please install these packages manually.')
             return
 
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.LINUX_CLANG)
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.LINUX_CBINDGEN)
 
 
+class NasmInstall(object):
+    def __init__(self, **kwargs):
+        pass
+
+    def ensure_nasm_packages(self, state_dir, checkout_root):
+        if is_non_x86_64():
+            print('Cannot install nasm from taskcluster.\n'
+                  'Please install this package manually.')
+            return
+
+        from mozboot import nasm
+        self.install_toolchain_artifact(state_dir, checkout_root, nasm.LINUX_NASM)
+
+
 class NodeInstall(object):
     def __init__(self, **kwargs):
         pass
 
     def ensure_node_packages(self, state_dir, checkout_root):
         if is_non_x86_64():
             print('Cannot install node package from taskcluster.\n'
                   'Please install this package manually.')
--- a/python/mozboot/mozboot/mozillabuild.py
+++ b/python/mozboot/mozboot/mozillabuild.py
@@ -84,16 +84,20 @@ class MozillaBuildBootstrapper(BaseBoots
                             'AArch64 device.  If you want to do artifact builds '
                             'instead, please choose the appropriate artifact build '
                             'option when beginning bootstrap.')
 
         from mozboot import stylo
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.WINDOWS_CLANG)
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.WINDOWS_CBINDGEN)
 
+    def ensure_nasm_packages(self, state_dir, checkout_root):
+        from mozboot import nasm
+        self.install_toolchain_artifact(state_dir, checkout_root, nasm.WINDOWS_NASM)
+
     def ensure_node_packages(self, state_dir, checkout_root):
         from mozboot import node
         # We don't have native aarch64 node available, but aarch64 windows
         # runs x86 binaries, so just use the x86 packages for such hosts.
         node_artifact = node.WIN32 if is_aarch64_host() else node.WIN64
         self.install_toolchain_artifact(
             state_dir, checkout_root, node_artifact)
 
new file mode 100644
--- /dev/null
+++ b/python/mozboot/mozboot/nasm.py
@@ -0,0 +1,8 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+WINDOWS_NASM = 'win64-nasm'
+LINUX_NASM = 'linux64-nasm'
--- a/python/mozboot/mozboot/openbsd.py
+++ b/python/mozboot/mozboot/openbsd.py
@@ -50,10 +50,14 @@ class OpenBSDBootstrapper(BaseBootstrapp
     def ensure_clang_static_analysis_package(self, state_dir, checkout_root):
         # TODO: we don't ship clang base static analysis for this platform
         pass
 
     def ensure_stylo_packages(self, state_dir, checkout_root):
         # Clang / llvm already installed as browser package
         self.run_as_root(['pkg_add', 'cbindgen'])
 
+    def ensure_nasm_packages(self, state_dir, checkout_root):
+        # installed via ensure_browser_packages
+        pass
+
     def ensure_node_packages(self, state_dir, checkout_root):
         self.run_as_root(['pkg_add', 'node'])
--- a/python/mozboot/mozboot/osx.py
+++ b/python/mozboot/mozboot/osx.py
@@ -524,16 +524,20 @@ class OSXBootstrapper(BaseBootstrapper):
     def ensure_stylo_packages(self, state_dir, checkout_root):
         from mozboot import stylo
         # We installed clang via homebrew earlier.  However, on Android, we're
         # seeing many compiler errors so we use our own toolchain clang.
         if 'mobile_android' in self.application:
             self.install_toolchain_artifact(state_dir, checkout_root, stylo.MACOS_CLANG)
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.MACOS_CBINDGEN)
 
+    def ensure_nasm_packages(self, state_dir, checkout_root):
+        # installed via ensure_browser_packages
+        pass
+
     def ensure_node_packages(self, state_dir, checkout_root):
         # XXX from necessary?
         from mozboot import node
         self.install_toolchain_artifact(state_dir, checkout_root, node.OSX)
 
     def install_homebrew(self):
         print(PACKAGE_MANAGER_INSTALL % ('Homebrew', 'Homebrew', 'Homebrew', 'brew'))
         bootstrap = urlopen(url=HOMEBREW_BOOTSTRAP, timeout=20).read()
--- a/python/mozboot/mozboot/windows.py
+++ b/python/mozboot/mozboot/windows.py
@@ -111,16 +111,20 @@ class WindowsBootstrapper(BaseBootstrapp
                             'AArch64 device.  If you want to do artifact builds '
                             'instead, please choose the appropriate artifact build '
                             'option when beginning bootstrap.')
 
         from mozboot import stylo
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.WINDOWS_CLANG)
         self.install_toolchain_artifact(state_dir, checkout_root, stylo.WINDOWS_CBINDGEN)
 
+    def ensure_nasm_packages(self, state_dir, checkout_root):
+        from mozboot import nasm
+        self.install_toolchain_artifact(state_dir, checkout_root, nasm.WINDOWS_NASM)
+
     def ensure_node_packages(self, state_dir, checkout_root):
         from mozboot import node
         # We don't have native aarch64 node available, but aarch64 windows
         # runs x86 binaries, so just use the x86 packages for such hosts.
         node_artifact = node.WIN32 if is_aarch64_host() else node.WIN64
         self.install_toolchain_artifact(
             state_dir, checkout_root, node_artifact)