Bug 1319332 - Derive the rust targets from the build host/target. r=froydnj
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 05 Oct 2017 16:10:45 +0900
changeset 385215 035d3125ca2f0e5eeb25a04be9faeaffd156546b
parent 385214 12293a4b3e23904229f8afd3e56805ffe072fde4
child 385216 e42bb7139c711615f264c58b601c23078686b086
push id32649
push userarchaeopteryx@coole-files.de
push dateTue, 10 Oct 2017 09:11:18 +0000
treeherdermozilla-central@87ca0b304342 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs1319332
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1319332 - Derive the rust targets from the build host/target. r=froydnj
build/moz.configure/init.configure
build/moz.configure/rust.configure
python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -492,17 +492,19 @@ option('--target', nargs=1,
        help='Define the system type where the resulting executables will be '
             'used')
 
 @imports(_from='mozbuild.configure.constants', _import='CPU')
 @imports(_from='mozbuild.configure.constants', _import='CPU_bitness')
 @imports(_from='mozbuild.configure.constants', _import='Endianness')
 @imports(_from='mozbuild.configure.constants', _import='Kernel')
 @imports(_from='mozbuild.configure.constants', _import='OS')
-def split_triplet(triplet):
+@imports(_from='__builtin__', _import='KeyError')
+@imports(_from='__builtin__', _import='ValueError')
+def split_triplet(triplet, allow_unknown=False):
     # The standard triplet is defined as
     #   CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
     # There is also a quartet form:
     #   CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM
     # But we can consider the "KERNEL-OPERATING_SYSTEM" as one.
     cpu, manufacturer, os = triplet.split('-', 2)
 
     # Autoconf uses config.sub to validate and canonicalize those triplets,
@@ -539,16 +541,18 @@ def split_triplet(triplet):
     elif os.startswith('freebsd'):
         canonical_os = canonical_kernel = 'FreeBSD'
     elif os.startswith('netbsd'):
         canonical_os = canonical_kernel = 'NetBSD'
     elif os.startswith('openbsd'):
         canonical_os = canonical_kernel = 'OpenBSD'
     elif os.startswith('solaris'):
         canonical_os = canonical_kernel = 'SunOS'
+    elif allow_unknown:
+        canonical_os = canonical_kernel = os
     else:
         die('Unknown OS: %s' % os)
 
     # The CPU granularity is probably not enough. Moving more things from
     # old-configure will tell us if we need more
     if cpu.endswith('86') or (cpu.startswith('i') and '86' in cpu):
         canonical_cpu = 'x86'
         endianness = 'little'
@@ -586,26 +590,40 @@ def split_triplet(triplet):
         canonical_cpu = 'mips64'
         endianness = 'little' if 'el' in cpu else 'big'
     elif cpu.startswith('aarch64'):
         canonical_cpu = 'aarch64'
         endianness = 'little'
     elif cpu == 'sh4':
         canonical_cpu = 'sh4'
         endianness = 'little'
+    elif allow_unknown:
+        canonical_cpu = cpu
+        endianness = 'unknown'
     else:
         die('Unknown CPU type: %s' % cpu)
 
+    def sanitize(cls, value):
+        try:
+            return cls(value)
+        except (KeyError, ValueError):
+            if allow_unknown:
+                return value
+            raise
+
+    def bitness(cpu):
+        return CPU_bitness[cpu]
+
     return namespace(
         alias=triplet,
-        cpu=CPU(canonical_cpu),
-        bitness=CPU_bitness[canonical_cpu],
-        kernel=Kernel(canonical_kernel),
-        os=OS(canonical_os),
-        endianness=Endianness(endianness),
+        cpu=sanitize(CPU, canonical_cpu),
+        bitness=sanitize(bitness, canonical_cpu),
+        kernel=sanitize(Kernel, canonical_kernel),
+        os=sanitize(OS, canonical_os),
+        endianness=sanitize(Endianness, endianness),
         raw_cpu=cpu,
         raw_os=os,
         # Toolchains, most notably for cross compilation may use cpu-os
         # prefixes.
         toolchain='%s-%s' % (cpu, os),
     )
 
 
--- a/build/moz.configure/rust.configure
+++ b/build/moz.configure/rust.configure
@@ -92,99 +92,97 @@ def rust_compiler(rustc_info, cargo_info
         To compile Rust language sources please install at least
         version {} of 'cargo' and make sure it is first in your path.
 
         You can verify this by typing 'cargo --version'.
         ''').format(version, cargo_min_version))
 
     return True
 
+
+@depends(rustc, when=rust_compiler)
+def rust_supported_targets(rustc):
+    out = check_cmd_output(rustc, '--print', 'target-list').splitlines()
+    # The os in the triplets used by rust may match the same OSes, in which
+    # case we need to check the raw_os instead.
+    per_os = {}
+    ambiguous = set()
+    per_raw_os = {}
+    for t in out:
+        t = split_triplet(t, allow_unknown=True)
+        key = (t.cpu, t.os)
+        if key in per_os:
+            previous = per_os[key]
+            per_raw_os[(previous.cpu, previous.raw_os)] = previous
+            del per_os[key]
+            ambiguous.add(key)
+        if key in ambiguous:
+            raw_os = t.raw_os
+            # split_triplet will return a raw_os of 'androideabi' for
+            # rust targets in the form cpu-linux-androideabi, but what
+            # we get from the build system is linux-androideabi, so
+            # normalize.
+            if raw_os == 'androideabi':
+                raw_os = 'linux-androideabi'
+            per_raw_os[(t.cpu, raw_os)] = t
+        else:
+            per_os[key] = t
+    return namespace(per_os=per_os, per_raw_os=per_raw_os)
+
+
 @template
 def rust_triple_alias(host_or_target):
     """Template defining the alias used for rustc's --target flag.
     `host_or_target` is either `host` or `target` (the @depends functions
     from init.configure).
     """
     assert host_or_target in (host, target)
 
-    @depends(rustc, host_or_target, building_with_gcc, when=rust_compiler)
+    @depends(rustc, host_or_target, building_with_gcc, rust_supported_targets,
+             when=rust_compiler)
     @imports('os')
     @imports('subprocess')
     @imports(_from='mozbuild.configure.util', _import='LineIO')
     @imports(_from='mozbuild.shellutil', _import='quote')
     @imports(_from='tempfile', _import='mkstemp')
     @imports(_from='textwrap', _import='dedent')
-    def rust_target(rustc, host_or_target, building_with_gcc):
+    def rust_target(rustc, host_or_target, building_with_gcc,
+                    rust_supported_targets):
         # Rust's --target options are similar to, but not exactly the same
         # as, the autoconf-derived targets we use.  An example would be that
         # Rust uses distinct target triples for targetting the GNU C++ ABI
         # and the MSVC C++ ABI on Win32, whereas autoconf has a single
         # triple and relies on the user to ensure that everything is
         # compiled for the appropriate ABI.  We need to perform appropriate
         # munging to get the correct option to rustc.
-        #
-        # The canonical list of targets supported can be derived from:
-        #
-        # https://github.com/rust-lang/rust/blob/master/src/librustc_back/target/mod.rs
+        # We correlate the autoconf-derived targets with the list of targets
+        # rustc gives us with --print target-list.
+        if host_or_target.kernel == 'WINNT':
+            if building_with_gcc:
+                host_or_target_os = 'windows-gnu'
+            else:
+                host_or_target_os = 'windows-msvc'
+            host_or_target_raw_os = host_or_target_os
+        else:
+            host_or_target_os = host_or_target.os
+            host_or_target_raw_os = host_or_target.raw_os
 
-        # Avoid having to write out os+kernel for all the platforms where
-        # they don't differ.
-        os_or_kernel = host_or_target.kernel if host_or_target.kernel == 'Linux' and host_or_target.os != 'Android' else host_or_target.os
-        # If we are targetting Windows but the compiler is gcc, we need to
-        # use a different rustc target
-        os_or_kernel = 'WINNT-MINGW' if building_with_gcc and host_or_target.kernel == 'WINNT' else os_or_kernel
-        rustc_target = {
-            # DragonFly
-            ('x86_64', 'DragonFly'): 'x86_64-unknown-dragonfly',
-            # FreeBSD
-            ('aarch64', 'FreeBSD'): 'aarch64-unknown-freebsd',
-            ('x86', 'FreeBSD'): 'i686-unknown-freebsd',
-            ('x86_64', 'FreeBSD'): 'x86_64-unknown-freebsd',
-            # NetBSD
-            ('sparc64', 'NetBSD'): 'sparc64-unknown-netbsd',
-            ('x86', 'NetBSD'): 'i686-unknown-netbsd',
-            ('x86_64', 'NetBSD'): 'x86_64-unknown-netbsd',
-            # OpenBSD
-            ('x86', 'OpenBSD'): 'i686-unknown-openbsd',
-            ('x86_64', 'OpenBSD'): 'x86_64-unknown-openbsd',
-            # Linux
-            ('aarch64', 'Linux'): 'aarch64-unknown-linux-gnu',
-            ('arm', 'Linux'): 'armv7-unknown-linux-gnueabihf',
-            ('sparc64', 'Linux'): 'sparc64-unknown-linux-gnu',
-            ('x86', 'Linux'): 'i686-unknown-linux-gnu',
-            ('x86_64', 'Linux'): 'x86_64-unknown-linux-gnu',
-            # OS X
-            ('x86', 'OSX'): 'i686-apple-darwin',
-            ('x86_64', 'OSX'): 'x86_64-apple-darwin',
-            # iOS
-            ('aarch64', 'iOS'): 'aarch64-apple-ios',
-            ('arm', 'iOS'): 'armv7s-apple-ios',
-            ('x86', 'iOS'): 'i386-apple-ios',
-            ('x86_64', 'iOS'): 'x86_64-apple-ios',
-            # Android
-            ('aarch64', 'Android'): 'aarch64-linux-android',
-            ('arm', 'Android'): 'armv7-linux-androideabi',
-            ('x86', 'Android'): 'i686-linux-android',
-            ('x86_64', 'Android'): 'x86_64-linux-android',
-            # Windows
-            ('x86', 'WINNT'): 'i686-pc-windows-msvc',
-            ('x86_64', 'WINNT'): 'x86_64-pc-windows-msvc',
-            ('x86', 'WINNT-MINGW'): 'i686-pc-windows-gnu',
-            ('x86_64', 'WINNT-MINGW'): 'x86_64-pc-windows-gnu',
-            # Solaris
-            ('x86_64', 'SunOS'): 'x86_64-sun-solaris',
-            ('sparc64', 'SunOS'): 'sparcv9-sun-solaris',
-        }.get((host_or_target.cpu, os_or_kernel), None)
+        rustc_target = rust_supported_targets.per_os.get(
+            (host_or_target.cpu, host_or_target_os))
+
+        if rustc_target is None:
+            rustc_target = rust_supported_targets.per_raw_os.get(
+                (host_or_target.cpu, host_or_target_raw_os))
 
         if rustc_target is None:
             die("Don't know how to translate {} for rustc".format(host_or_target.alias))
 
         # Check to see whether our rustc has a reasonably functional stdlib
         # for our chosen target.
-        target_arg = '--target=' + rustc_target
+        target_arg = '--target=' + rustc_target.alias
         in_fd, in_path = mkstemp(prefix='conftest', suffix='.rs')
         out_fd, out_path = mkstemp(prefix='conftest', suffix='.rlib')
         os.close(out_fd)
         try:
             source = 'pub extern fn hello() { println!("Hello world"); }'
             log.debug('Creating `%s` with content:', in_path)
             with LineIO(lambda l: log.debug('| %s', l)) as out:
                 out.write(source)
@@ -201,26 +199,26 @@ def rust_triple_alias(host_or_target):
             ]
             def failed():
                 die(dedent('''\
                 Cannot compile for {} with {}
                 The target may be unsupported, or you may not have
                 a rust std library for that target installed. Try:
 
                   rustup target add {}
-                '''.format(host_or_target.alias, rustc, rustc_target)))
+                '''.format(host_or_target.alias, rustc, rustc_target.alias)))
             check_cmd_output(*cmd, onerror=failed)
             if not os.path.exists(out_path) or os.path.getsize(out_path) == 0:
                 failed()
         finally:
             os.remove(in_path)
             os.remove(out_path)
 
         # This target is usable.
-        return rustc_target
+        return rustc_target.alias
 
     return rust_target
 
 rust_target_triple = rust_triple_alias(target)
 rust_host_triple = rust_triple_alias(host)
 
 set_config('RUST_TARGET', rust_target_triple)
 set_config('RUST_HOST_TARGET', rust_host_triple)
--- a/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
@@ -1403,10 +1403,170 @@ class OpenBSDToolchainTest(BaseToolchain
 
     def test_gcc(self):
         self.do_toolchain_test(self.PATHS, {
             'c_compiler': self.GCC_4_9_RESULT,
             'cxx_compiler': self.GXX_4_9_RESULT,
         })
 
 
+class RustTest(BaseConfigureTest):
+    def invoke_cargo(self, stdin, args):
+        if args == ('--version', '--verbose'):
+            return 0, 'cargo 2.0\nrelease: 2.0', ''
+        raise NotImplementedError('unsupported arguments')
+
+    def invoke_rustc(self, stdin, args):
+        if args == ('--version', '--verbose'):
+            return 0, 'rustc 2.0\nrelease: 2.0', ''
+        if args == ('--print', 'target-list'):
+            # Raw list returned by rustc version 1.19, + ios, which somehow
+            # don't appear in the default list.
+            # https://github.com/rust-lang/rust/issues/36156
+            rust_targets = [
+                'aarch64-apple-ios',
+                'aarch64-linux-android',
+                'aarch64-unknown-freebsd',
+                'aarch64-unknown-fuchsia',
+                'aarch64-unknown-linux-gnu',
+                'arm-linux-androideabi',
+                'arm-unknown-linux-gnueabi',
+                'arm-unknown-linux-gnueabihf',
+                'arm-unknown-linux-musleabi',
+                'arm-unknown-linux-musleabihf',
+                'armv5te-unknown-linux-gnueabi',
+                'armv7-linux-androideabi',
+                'armv7-unknown-linux-gnueabihf',
+                'armv7-unknown-linux-musleabihf',
+                'armv7s-apple-ios',
+                'asmjs-unknown-emscripten',
+                'i386-apple-ios',
+                'i586-pc-windows-msvc',
+                'i586-unknown-linux-gnu',
+                'i686-apple-darwin',
+                'i686-linux-android',
+                'i686-pc-windows-gnu',
+                'i686-pc-windows-msvc',
+                'i686-unknown-dragonfly',
+                'i686-unknown-freebsd',
+                'i686-unknown-haiku',
+                'i686-unknown-linux-gnu',
+                'i686-unknown-linux-musl',
+                'i686-unknown-netbsd',
+                'i686-unknown-openbsd',
+                'le32-unknown-nacl',
+                'mips-unknown-linux-gnu',
+                'mips-unknown-linux-musl',
+                'mips-unknown-linux-uclibc',
+                'mips64-unknown-linux-gnuabi64',
+                'mips64el-unknown-linux-gnuabi64',
+                'mipsel-unknown-linux-gnu',
+                'mipsel-unknown-linux-musl',
+                'mipsel-unknown-linux-uclibc',
+                'powerpc-unknown-linux-gnu',
+                'powerpc64-unknown-linux-gnu',
+                'powerpc64le-unknown-linux-gnu',
+                's390x-unknown-linux-gnu',
+                'sparc64-unknown-linux-gnu',
+                'sparc64-unknown-netbsd',
+                'sparcv9-sun-solaris',
+                'thumbv6m-none-eabi',
+                'thumbv7em-none-eabi',
+                'thumbv7em-none-eabihf',
+                'thumbv7m-none-eabi',
+                'wasm32-unknown-emscripten',
+                'x86_64-apple-darwin',
+                'x86_64-apple-ios',
+                'x86_64-linux-android',
+                'x86_64-pc-windows-gnu',
+                'x86_64-pc-windows-msvc',
+                'x86_64-rumprun-netbsd',
+                'x86_64-sun-solaris',
+                'x86_64-unknown-bitrig',
+                'x86_64-unknown-dragonfly',
+                'x86_64-unknown-freebsd',
+                'x86_64-unknown-fuchsia',
+                'x86_64-unknown-haiku',
+                'x86_64-unknown-linux-gnu',
+                'x86_64-unknown-linux-musl',
+                'x86_64-unknown-netbsd',
+                'x86_64-unknown-openbsd',
+                'x86_64-unknown-redox',
+            ]
+            return 0, '\n'.join(rust_targets), ''
+        if (len(args) == 6 and args[:2] == ('--crate-type', 'staticlib') and
+            args[2].startswith('--target=') and args[3] == '-o'):
+            with open(args[4], 'w') as fh:
+                fh.write('foo')
+            return 0, '', ''
+        raise NotImplementedError('unsupported arguments')
+
+    def get_rust_target(self, target, building_with_gcc=True):
+        environ = {
+            'PATH': os.pathsep.join(
+                mozpath.abspath(p) for p in ('/bin', '/usr/bin')),
+        }
+
+        paths = {
+            mozpath.abspath('/usr/bin/cargo'): self.invoke_cargo,
+            mozpath.abspath('/usr/bin/rustc'): self.invoke_rustc,
+        }
+
+        self.TARGET = target
+        sandbox = self.get_sandbox(paths, {}, [], environ)
+
+        # Trick the sandbox into not running the target compiler check
+        dep = sandbox._depends[sandbox['building_with_gcc']]
+        getattr(sandbox, '__value_for_depends')[(dep, False)] = \
+            building_with_gcc
+        return sandbox._value_for(sandbox['rust_target_triple'])
+
+    def test_rust_target(self):
+        # Cases where the output of config.sub matches a rust target
+        for straightforward in (
+            'x86_64-unknown-dragonfly',
+            'aarch64-unknown-freebsd',
+            'i686-unknown-freebsd',
+            'x86_64-unknown-freebsd',
+            'sparc64-unknown-netbsd',
+            'i686-unknown-netbsd',
+            'x86_64-unknown-netbsd',
+            'i686-unknown-openbsd',
+            'x86_64-unknown-openbsd',
+            'aarch64-unknown-linux-gnu',
+            'armv7-unknown-linux-gnueabihf',
+            'sparc64-unknown-linux-gnu',
+            'i686-unknown-linux-gnu',
+            'i686-apple-darwin',
+            'x86_64-apple-darwin',
+            'aarch64-apple-ios',
+            'armv7s-apple-ios',
+            'i386-apple-ios',
+            'x86_64-apple-ios',
+        ):
+            self.assertEqual(self.get_rust_target(straightforward), straightforward)
+
+        # Cases where the output of config.sub is different
+        for autoconf, rust in (
+            ('aarch64-unknown-linux-android', 'aarch64-linux-android'),
+            ('arm-unknown-linux-androideabi', 'armv7-linux-androideabi'),
+            ('armv7-unknown-linux-androideabi', 'armv7-linux-androideabi'),
+            ('i386-unknown-linux-android', 'i686-linux-android'),
+            ('i686-unknown-linux-android', 'i686-linux-android'),
+            ('x86_64-unknown-linux-android', 'x86_64-linux-android'),
+            ('x86_64-pc-linux-gnu', 'x86_64-unknown-linux-gnu'),
+            ('sparcv9-sun-solaris2', 'sparcv9-sun-solaris'),
+            ('x86_64-sun-solaris2', 'x86_64-sun-solaris'),
+        ):
+            self.assertEqual(self.get_rust_target(autoconf), rust)
+
+        # Windows
+        for autoconf, building_with_gcc, rust in (
+            ('i686-pc-mingw32', False, 'i686-pc-windows-msvc'),
+            ('x86_64-pc-mingw32', False, 'x86_64-pc-windows-msvc'),
+            ('i686-pc-mingw32', True, 'i686-pc-windows-gnu'),
+            ('x86_64-pc-mingw32', True, 'x86_64-pc-windows-gnu'),
+        ):
+            self.assertEqual(self.get_rust_target(autoconf, building_with_gcc), rust)
+
+
 if __name__ == '__main__':
     main()