Bug 1407678 - Make windows_toolchain.py support VS2017. r=mshal DONTBUILD
authorDavid Major <dmajor@mozilla.com>
Thu, 02 Nov 2017 11:10:39 -0400
changeset 443131 3f56c67e2a2ab7404641d19dec37254c2523d475
parent 443106 d3a0101e3936adcd5eb90c48c4fd6d768c6fabe3
child 443132 831a02dbf05dc05facbae533ca474d2c4dbb7518
push id1618
push userCallek@gmail.com
push dateThu, 11 Jan 2018 17:45:48 +0000
treeherdermozilla-release@882ca853e05a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmshal
bugs1407678
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1407678 - Make windows_toolchain.py support VS2017. r=mshal DONTBUILD
build/docs/toolchains.rst
build/windows_toolchain.py
--- a/build/docs/toolchains.rst
+++ b/build/docs/toolchains.rst
@@ -46,17 +46,17 @@ 2. Select ``Programming Languages`` -> `
 3. Under ``Windows and Web Development`` uncheck everything except
    ``Universal Windows App Development Tools`` and the items under it
    (should be ``Tools (1.3.1)...`` and the ``Windows 10 SDK``).
 
 Once Visual Studio 2015 Community has been installed, from a checkout
 of mozilla-central, run something like the following to produce a ZIP
 archive::
 
-   $ ./mach python build/windows_toolchain.py create-zip vs2015u3
+   $ ./mach python build/windows_toolchain.py create-zip vs2017_15.4.2
 
 The produced archive will be the argument to ``create-zip`` + ``.zip``.
 
 Firefox for Android with Gradle
 ===============================
 
 To build Firefox for Android with Gradle in automation, archives
 containing both the Gradle executable and a Maven repository
--- a/build/windows_toolchain.py
+++ b/build/windows_toolchain.py
@@ -19,155 +19,171 @@ import sys
 from mozpack.files import (
     FileFinder,
 )
 from mozpack.mozjar import (
     JarWriter,
 )
 import mozpack.path as mozpath
 
+SDK_RELEASE = '10.0.15063.0'
 
-# mozpack.match patterns for files under "Microsoft Visual Studio 14.0".
-VS_PATTERNS = [
-    {
-        'pattern': 'DIA SDK/bin/**',
-        'ignore': (
-            'DIA SDK/bin/arm/**',
-        ),
-    },
-    {
-        'pattern': 'DIA SDK/idl/**',
-    },
-    {
-        'pattern': 'DIA SDK/include/**',
-    },
+PATTERNS = [
     {
-        'pattern': 'DIA SDK/lib/**',
-        'ignore': (
-            'DIA SDK/lib/arm/**',
-        ),
-    },
-    # ATL is needed by Breakpad.
-    {
-        'pattern': 'VC/atlmfc/include/**',
-    },
-    {
-        'pattern': 'VC/atlmfc/lib/atls.*',
-    },
-    {
-        'pattern': 'VC/atlmfc/lib/amd64/atls.*',
-    },
-    {
-        'pattern': 'VC/bin/**',
-        # We only care about compiling on amd64 for amd64 or x86 targets.
-        'ignore': (
-            'VC/bin/amd64_arm/**',
-            'VC/bin/arm/**',
-            'VC/bin/x86_arm/**',
-            'VC/bin/x86_amd64/**',
-        ),
-    },
-    {
-        'pattern': 'VC/include/**',
+        'srcdir': '%(vs_path)s/DIA SDK',
+        'dstdir': 'DIA SDK',
+        'files': [
+            {
+                'pattern': 'bin/**',
+                'ignore': (
+                    'bin/arm/**',
+                ),
+            },
+            {
+                'pattern': 'idl/**',
+            },
+            {
+                'pattern': 'include/**',
+            },
+            {
+                'pattern': 'lib/**',
+                'ignore': (
+                    'lib/arm/**',
+                ),
+            },
+        ],
     },
     {
-        'pattern': 'VC/lib/**',
-        'ignore': (
-            'VC/lib/arm/**',
-            'VC/lib/onecore/**',
-            'VC/lib/store/**',
-        ),
-    },
-    {
-        'pattern': 'VC/redist/x64/Microsoft.VC140.CRT/**',
-    },
-    {
-        'pattern': 'VC/redist/x86/Microsoft.VC140.CRT/**',
-    },
-]
-
-SDK_RELEASE = '10.0.14393.0'
-
-# Files from the Windows 10 SDK to install.
-SDK_PATTERNS = [
-    {
-        'pattern': 'bin/x64/**',
+        'srcdir': '%(vs_path)s/VC/Tools/MSVC/14.11.25503',
+        'dstdir': 'VC',
+        'files': [
+            # ATL is needed by Breakpad.
+            {
+                'pattern': 'atlmfc/include/**',
+            },
+            {
+                'pattern': 'atlmfc/lib/x86/atls.*',
+            },
+            {
+                'pattern': 'atlmfc/lib/x64/atls.*',
+            },
+            {
+                'pattern': 'bin/Hostx64/**',
+            },
+            # 32-bit PGO-instrumented builds require 32-bit pgort140.dll.
+            {
+                'pattern': 'bin/Hostx86/x86/pgort140.dll',
+            },
+            {
+                'pattern': 'include/**',
+            },
+            {
+                'pattern': 'lib/**',
+                'ignore': (
+                    'lib/onecore/**',
+                    'lib/x64/store/**',
+                    'lib/x86/store/**',
+                ),
+            },
+        ],
     },
     {
-        'pattern': 'Include/%s/**' % SDK_RELEASE,
-    },
-    {
-        'pattern': 'Lib/%s/ucrt/x64/**' % SDK_RELEASE,
-    },
-    {
-        'pattern': 'Lib/%s/ucrt/x86/**' % SDK_RELEASE,
+        'srcdir': '%(vs_path)s/VC/Redist/MSVC/14.11.25325',
+        'dstdir': 'VC/redist',
+        'files': [
+            {
+                'pattern': 'x64/Microsoft.VC141.CRT/**',
+            },
+            {
+                'pattern': 'x86/Microsoft.VC141.CRT/**',
+            },
+        ],
     },
     {
-        'pattern': 'Lib/%s/um/x64/**' % SDK_RELEASE,
-    },
-    {
-        'pattern': 'Lib/%s/um/x86/**' % SDK_RELEASE,
-    },
-    {
-        'pattern': 'Redist/D3D/**',
-    },
-    {
-        'pattern': 'Redist/ucrt/DLLs/x64/**',
-    },
-    {
-        'pattern': 'Redist/ucrt/DLLs/x86/**',
+        'srcdir': '%(sdk_path)s',
+        'dstdir': 'SDK',
+        'files': [
+            {
+                'pattern': 'bin/%s/x64/**' % SDK_RELEASE,
+            },
+            {
+                'pattern': 'Include/%s/**' % SDK_RELEASE,
+            },
+            {
+                'pattern': 'Lib/%s/ucrt/x64/**' % SDK_RELEASE,
+            },
+            {
+                'pattern': 'Lib/%s/ucrt/x86/**' % SDK_RELEASE,
+            },
+            {
+                'pattern': 'Lib/%s/um/x64/**' % SDK_RELEASE,
+            },
+            {
+                'pattern': 'Lib/%s/um/x86/**' % SDK_RELEASE,
+            },
+            {
+                'pattern': 'Redist/D3D/**',
+            },
+            {
+                'pattern': 'Redist/ucrt/DLLs/x64/**',
+            },
+            {
+                'pattern': 'Redist/ucrt/DLLs/x86/**',
+            },
+        ],
     },
 ]
 
 
 def find_vs_paths():
     """Resolve source locations of files.
 
     Returns a 2-tuple of (Visual Studio Path, SDK Path).
     """
     pf = os.environ.get('ProgramFiles(x86)')
     if not pf:
         raise Exception('No "ProgramFiles(x86)" environment variable. '
                         'Not running on 64-bit Windows?')
 
-    vs_path = os.path.join(pf, 'Microsoft Visual Studio 14.0')
+    vs_path = os.path.join(pf, 'Microsoft Visual Studio', '2017', 'Community')
     if not os.path.exists(vs_path):
-        raise Exception('%s does not exist; Visual Studio 2015 not installed?' %
+        raise Exception('%s does not exist; Visual Studio 2017 not installed?' %
                         vs_path)
 
     sdk_path = os.path.join(pf, 'Windows Kits', '10')
     if not os.path.exists(sdk_path):
         raise Exception('%s does not exist; Windows 10 SDK not installed?' %
                         sdk_path)
 
+    sdk_fullver_path = os.path.join(sdk_path, 'Include', SDK_RELEASE)
+    if not os.path.exists(sdk_fullver_path):
+        raise Exception('%s does not exist; Wrong SDK version installed?' %
+                        sdk_fullver_path)
+
     return vs_path, sdk_path
 
 
 def resolve_files():
     """Resolve the files that constitute a standalone toolchain.
 
     This is a generator of (dest path, file) where the destination
     path is relative and the file instance is a BaseFile from mozpack.
     """
     vs_path, sdk_path = find_vs_paths()
 
-    for entry in VS_PATTERNS:
-        finder = FileFinder(vs_path, ignore=entry.get('ignore', []))
-        for p, f in finder.find(entry['pattern']):
-            assert p.startswith(('VC/', 'DIA SDK/'))
-
-            yield p.encode('utf-8'), f
-
-    for entry in SDK_PATTERNS:
-        finder = FileFinder(sdk_path, ignore=entry.get('ignore', []))
-        for p, f in finder.find(entry['pattern']):
-            relpath = 'SDK/%s' % p
-
-            yield relpath.encode('utf-8'), f
-
+    for entry in PATTERNS:
+        fullpath = entry['srcdir'] % {
+            'vs_path': vs_path,
+            'sdk_path': sdk_path,
+        }
+        for pattern in entry['files']:
+            finder = FileFinder(fullpath, ignore=pattern.get('ignore', []))
+            for p, f in finder.find(pattern['pattern']):
+                dstpath = '%s/%s' % (entry['dstdir'], p)
+                yield dstpath.encode('utf-8'), f
 
 def resolve_files_and_hash(manifest):
     """Resolve files and hash their data.
 
     This is a generator of 3-tuples of (relpath, data, mode).
 
     As data is read, the manifest is populated with metadata.
     Keys are set to the relative file path. Values are 2-tuples
@@ -196,17 +212,17 @@ def format_manifest(manifest):
     return b'\n'.join(sha256_lines)
 
 
 def write_zip(zip_path, prefix=None):
     """Write toolchain data to a zip file."""
     if isinstance(prefix, unicode): # noqa Special case for Python 2
         prefix = prefix.encode('utf-8')
 
-    with JarWriter(file=zip_path, optimize=False, compress=5) as zip:
+    with JarWriter(file=zip_path, optimize=False, compress_level=5) as zip:
         manifest = {}
         for p, data, mode in resolve_files_and_hash(manifest):
             print(p)
             if prefix:
                 p = mozpath.join(prefix, p)
 
             zip.add(p, data, mode=mode)