Bug 1560044 - Always use a SDK path on macOS. r=nalexander
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 03 Jul 2019 22:11:05 +0000
changeset 544030 3fd5b373540bef90c66d7ab389c696b45893c26c
parent 544029 9114bef9e25a145de8223c42cb3555fd5bab942e
child 544031 715b1861e5ac867e1ad5b3cc2b4ac78aa1cdb472
push id2131
push userffxbld-merge
push dateMon, 26 Aug 2019 18:30:20 +0000
treeherdermozilla-release@b19ffb3ca153 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1560044
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1560044 - Always use a SDK path on macOS. r=nalexander The SDK headers may not be installed in /usr/include. The usual response has been to have people run e.g. `open /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg` which is not really sustainable. This makes builds that happen on a macOS host try to detect their SDK and use that as a default for --with-macos-sdk, which has the side effect of enabling the SDK version check in that configuration. Differential Revision: https://phabricator.services.mozilla.com/D36558
build/moz.configure/toolchain.configure
python/mozbuild/mozbuild/test/configure/fake_macos_sdk/SDKSettings.plist
python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py
--- a/build/moz.configure/toolchain.configure
+++ b/build/moz.configure/toolchain.configure
@@ -131,45 +131,56 @@ def host_is_osx(host):
 
 
 with only_when(host_is_osx | target_is_osx):
     # MacOS SDK
     # =========
     js_option('--with-macos-sdk', env='MACOS_SDK_DIR', nargs=1,
            help='Location of platform SDK to use')
 
-    @depends_if('--with-macos-sdk')
+    @depends('--with-macos-sdk', host)
     @imports(_from='os.path', _import='isdir')
     @imports(_from='biplist', _import='readPlist')
-    def macos_sdk(value):
+    def macos_sdk(sdk, host):
         sdk_min_version = Version('10.11')
         sdk_max_version = Version('10.14')
 
-        if not isdir(value[0]):
+        if sdk:
+            sdk = sdk[0]
+        elif host.os == 'OSX':
+            sdk = check_cmd_output('xcrun', '--show-sdk-path', onerror=lambda: '').rstrip()
+            if not sdk:
+                die('Could not find the macOS SDK. Please use --with-macos-sdk to give '
+                    'the path to a macOS SDK.')
+        else:
+            die('Need a macOS SDK when targeting macOS. Please use --with-macos-sdk '
+                'to give the path to a macOS SDK.')
+
+        if not isdir(sdk):
             die('SDK not found in %s. When using --with-macos-sdk, you must specify a '
                 'valid SDK. SDKs are installed when the optional cross-development '
                 'tools are selected during the Xcode/Developer Tools installation.'
-                % value[0])
-        obj = readPlist(os.path.join(value[0], 'SDKSettings.plist'))
+                % sdk)
+        obj = readPlist(os.path.join(sdk, 'SDKSettings.plist'))
         if not obj:
-            die('Error parsing SDKSettings.plist in the SDK directory: %s' % value[0])
+            die('Error parsing SDKSettings.plist in the SDK directory: %s' % sdk)
         if 'Version' not in obj:
-            die('Error finding Version information in SDKSettings.plist from the SDK: %s' % value[0])
+            die('Error finding Version information in SDKSettings.plist from the SDK: %s' % sdk)
         version = Version(obj['Version'])
         if version < sdk_min_version:
             die('SDK version "%s" is too old. Please upgrade to at least %s. '
                 'You may need to point to it using --with-macos-sdk=<path> in your '
                 'mozconfig. Various SDK versions are available from '
                 'https://github.com/phracker/MacOSX-SDKs' % (version, sdk_min_version))
         if version > sdk_max_version:
             die('SDK version "%s" is unsupported. Please downgrade to version '
                 '%s. You may need to point to it using --with-macos-sdk=<path> in '
                 'your mozconfig. Various SDK versions are available from '
                 'https://github.com/phracker/MacOSX-SDKs' % (version, sdk_max_version))
-        return value[0]
+        return sdk
 
     set_config('MACOS_SDK_DIR', macos_sdk)
 
 
 with only_when(target_is_osx):
     with only_when(cross_compiling):
         option('--with-macos-private-frameworks',
                env="MACOS_PRIVATE_FRAMEWORKS_DIR", nargs=1,
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/configure/fake_macos_sdk/SDKSettings.plist
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+        <key>Version</key>
+        <string>10.14</string>
+</dict>
+</plist>
--- a/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
@@ -16,16 +16,17 @@ from mozbuild.configure.util import Vers
 from mozbuild.util import (
     memoize,
     ReadOnlyNamespace,
 )
 from mozpack import path as mozpath
 from test_toolchain_helpers import (
     FakeCompiler,
     CompilerResult,
+    PrependFlags,
 )
 
 
 DEFAULT_C99 = {
     '__STDC_VERSION__': '199901L',
 }
 
 DEFAULT_C11 = {
@@ -791,44 +792,55 @@ class LinuxX86_64CrossToolchainTest(Base
             'cxx_compiler': self.DEFAULT_CLANGXX_RESULT + {
                 'flags': ['-m64']
             },
             'host_c_compiler': self.DEFAULT_CLANG_RESULT,
             'host_cxx_compiler': self.DEFAULT_CLANGXX_RESULT,
         })
 
 
+def xcrun(stdin, args):
+    if args == ('--show-sdk-path',):
+        return 0, os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                               'fake_macos_sdk'), ''
+    raise NotImplementedError()
+
+
 class OSXToolchainTest(BaseToolchainTest):
     HOST = 'x86_64-apple-darwin11.2.0'
     PATHS = {
         '/usr/bin/gcc-5': GCC_5 + GCC_PLATFORM_X86_64_OSX,
         '/usr/bin/g++-5': GXX_5 + GCC_PLATFORM_X86_64_OSX,
         '/usr/bin/gcc-7': GCC_7 + GCC_PLATFORM_X86_64_OSX,
         '/usr/bin/g++-7': GXX_7 + GCC_PLATFORM_X86_64_OSX,
         '/usr/bin/clang': DEFAULT_CLANG + CLANG_PLATFORM_X86_64_OSX,
         '/usr/bin/clang++': DEFAULT_CLANGXX + CLANG_PLATFORM_X86_64_OSX,
         '/usr/bin/clang-4.0': CLANG_4_0 + CLANG_PLATFORM_X86_64_OSX,
         '/usr/bin/clang++-4.0': CLANGXX_4_0 + CLANG_PLATFORM_X86_64_OSX,
         '/usr/bin/clang-3.3': CLANG_3_3 + CLANG_PLATFORM_X86_64_OSX,
         '/usr/bin/clang++-3.3': CLANGXX_3_3 + CLANG_PLATFORM_X86_64_OSX,
+        '/usr/bin/xcrun': xcrun,
     }
     CLANG_3_3_RESULT = LinuxToolchainTest.CLANG_3_3_RESULT
     CLANGXX_3_3_RESULT = LinuxToolchainTest.CLANGXX_3_3_RESULT
     DEFAULT_CLANG_RESULT = LinuxToolchainTest.DEFAULT_CLANG_RESULT
     DEFAULT_CLANGXX_RESULT = LinuxToolchainTest.DEFAULT_CLANGXX_RESULT
     GCC_5_RESULT = LinuxToolchainTest.GCC_5_RESULT
     GXX_5_RESULT = LinuxToolchainTest.GXX_5_RESULT
     GCC_7_RESULT = LinuxToolchainTest.GCC_7_RESULT
     GXX_7_RESULT = LinuxToolchainTest.GXX_7_RESULT
+    SYSROOT_FLAGS = {
+        'flags': PrependFlags(['-isysroot', xcrun('', ('--show-sdk-path',))[1]]),
+    }
 
     def test_clang(self):
         # We only try clang because gcc is known not to work.
         self.do_toolchain_test(self.PATHS, {
-            'c_compiler': self.DEFAULT_CLANG_RESULT,
-            'cxx_compiler': self.DEFAULT_CLANGXX_RESULT,
+            'c_compiler': self.DEFAULT_CLANG_RESULT + self.SYSROOT_FLAGS,
+            'cxx_compiler': self.DEFAULT_CLANGXX_RESULT + self.SYSROOT_FLAGS,
         })
 
     def test_not_gcc(self):
         # We won't pick GCC if it's the only thing available.
         paths = {
             k: v for k, v in self.PATHS.iteritems()
             if os.path.basename(k) not in ('clang', 'clang++')
         }
@@ -844,18 +856,18 @@ class OSXToolchainTest(BaseToolchainTest
         }, environ={
             'CC': 'clang-3.3',
             'CXX': 'clang++-3.3',
         })
 
     def test_forced_gcc(self):
         # GCC can still be forced if the user really wants it.
         self.do_toolchain_test(self.PATHS, {
-            'c_compiler': self.GCC_7_RESULT,
-            'cxx_compiler': self.GXX_7_RESULT,
+            'c_compiler': self.GCC_7_RESULT + self.SYSROOT_FLAGS,
+            'cxx_compiler': self.GXX_7_RESULT + self.SYSROOT_FLAGS,
         }, environ={
             'CC': 'gcc-7',
             'CXX': 'g++-7',
         })
 
     def test_forced_unsupported_gcc(self):
         self.do_toolchain_test(self.PATHS, {
             'c_compiler': self.GCC_5_RESULT,
@@ -1317,35 +1329,35 @@ class LinuxCrossCompileToolchainTest(Bas
 class OSXCrossToolchainTest(BaseToolchainTest):
     TARGET = 'i686-apple-darwin11.2.0'
     PATHS = LinuxToolchainTest.PATHS
     DEFAULT_CLANG_RESULT = LinuxToolchainTest.DEFAULT_CLANG_RESULT
     DEFAULT_CLANGXX_RESULT = LinuxToolchainTest.DEFAULT_CLANGXX_RESULT
 
     def test_osx_cross(self):
         self.do_toolchain_test(self.PATHS, {
-            'c_compiler': self.DEFAULT_CLANG_RESULT + {
+            'c_compiler': self.DEFAULT_CLANG_RESULT + OSXToolchainTest.SYSROOT_FLAGS + {
                 'flags': ['--target=i686-apple-darwin11.2.0'],
             },
-            'cxx_compiler': self.DEFAULT_CLANGXX_RESULT + {
+            'cxx_compiler': self.DEFAULT_CLANGXX_RESULT + OSXToolchainTest.SYSROOT_FLAGS + {
                 'flags': ['--target=i686-apple-darwin11.2.0'],
             },
             'host_c_compiler': self.DEFAULT_CLANG_RESULT,
             'host_cxx_compiler': self.DEFAULT_CLANGXX_RESULT,
         }, environ={
             'CC': 'clang',
-        })
+        }, args=['--with-macos-sdk=%s' % OSXToolchainTest.SYSROOT_FLAGS['flags'][1]])
 
     def test_cannot_osx_cross(self):
         self.do_toolchain_test(self.PATHS, {
             'c_compiler': 'Target C compiler target kernel (Linux) does not '
                           'match --target kernel (Darwin)',
         }, environ={
             'CC': 'gcc',
-        })
+        }, args=['--with-macos-sdk=%s' % OSXToolchainTest.SYSROOT_FLAGS['flags'][1]])
 
 
 class WindowsCrossToolchainTest(BaseToolchainTest):
     TARGET = 'x86_64-pc-mingw32'
     DEFAULT_CLANG_RESULT = LinuxToolchainTest.DEFAULT_CLANG_RESULT
     DEFAULT_CLANGXX_RESULT = LinuxToolchainTest.DEFAULT_CLANGXX_RESULT
 
     def test_clang_cl_cross(self):
--- a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py
+++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py
@@ -168,18 +168,32 @@ class FakeCompiler(dict):
     def __init__(self, *definitions):
         for definition in definitions:
             if all(not isinstance(d, dict) for d in definition.itervalues()):
                 definition = {None: definition}
             for key, value in definition.iteritems():
                 self.setdefault(key, {}).update(value)
 
     def __call__(self, stdin, args):
-        files = [arg for arg in args if not arg.startswith('-')]
-        flags = [arg for arg in args if arg.startswith('-')]
+        files = []
+        flags = []
+        args = iter(args)
+        while True:
+            arg = next(args, None)
+            if arg is None:
+                break
+            if arg.startswith('-'):
+                # Ignore -isysroot and the argument that follows it.
+                if arg == '-isysroot':
+                    next(args, None)
+                else:
+                    flags.append(arg)
+            else:
+                files.append(arg)
+
         if '-E' in flags:
             assert len(files) == 1
             file = files[0]
             pp = CompilerPreprocessor(self[None])
 
             def apply_defn(defn):
                 for k, v in defn.iteritems():
                     if v is False:
@@ -342,16 +356,22 @@ class TestFakeCompiler(unittest.TestCase
                 'D': 5,
             },
             '-bar': {
                 'E': 6,
             },
         })
 
 
+class PrependFlags(list):
+    '''Wrapper to allow to Prepend to flags instead of appending, in
+    CompilerResult.
+    '''
+
+
 class CompilerResult(ReadOnlyNamespace):
     '''Helper of convenience to manipulate toolchain results in unit tests
 
     When adding a dict, the result is a new CompilerResult with the values
     from the dict replacing those from the CompilerResult, except for `flags`,
     where the value from the dict extends the `flags` in `self`.
     '''
 
@@ -370,17 +390,21 @@ class CompilerResult(ReadOnlyNamespace):
             language=language,
         )
 
     def __add__(self, other):
         assert isinstance(other, dict)
         result = copy.deepcopy(self.__dict__)
         for k, v in other.iteritems():
             if k == 'flags':
-                result.setdefault(k, []).extend(v)
+                flags = result.setdefault(k, [])
+                if isinstance(v, PrependFlags):
+                    flags[:0] = v
+                else:
+                    flags.extend(v)
             else:
                 result[k] = v
         return CompilerResult(**result)
 
 
 class TestCompilerResult(unittest.TestCase):
     def test_compiler_result(self):
         result = CompilerResult()