bug 1329320 - replace makecab with rust-makecab in symbolstore. r=gps
authorTed Mielczarek <ted@mielczarek.org>
Fri, 06 Jan 2017 05:57:10 -0500
changeset 328759 e31b6b05ba5b
parent 328758 06a71c58934a
child 328760 48517fcee770
push id31187
push userkwierso@gmail.com
push dateWed, 11 Jan 2017 01:56:54 +0000
treeherdermozilla-central@b079c9833e3e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1329320
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1329320 - replace makecab with rust-makecab in symbolstore. r=gps It turns out that running makecab to compress PDB files takes a significant amount of time in the buildsymbols step. I wrote an implementation of makecab in Rust that implements only the subset of features we use and it's significantly faster: https://github.com/luser/rust-makecab This patch adds a makecab check to moz.configure, adds a release build of the makecab binary to the Windows tooltool manifests, points the build at it from mozconfig.win-common, and changes symbolstore.py to use MAKECAB from substs instead of calling `makecab.exe` directly. MozReview-Commit-ID: 76FHLIZFCXS
browser/config/tooltool-manifests/win32/clang.manifest
browser/config/tooltool-manifests/win32/releng.manifest
browser/config/tooltool-manifests/win64/clang.manifest
browser/config/tooltool-manifests/win64/releng.manifest
build/moz.configure/windows.configure
build/mozconfig.win-common
toolkit/crashreporter/tools/symbolstore.py
toolkit/crashreporter/tools/unit-symbolstore.py
--- a/browser/config/tooltool-manifests/win32/clang.manifest
+++ b/browser/config/tooltool-manifests/win32/clang.manifest
@@ -32,10 +32,18 @@
 },
 {
 "version": "clang 4.0pre/r286542",
 "size": 222604502,
 "digest": "cea6119131adb66e0b7ec5030b00922ac95e4e97249fcab9561a848ea60b7f80536c9171a07136afcb79decbcdb20099a5e7ee493013710b8ba5ae072ad40851",
 "algorithm": "sha512",
 "filename": "clang.tar.bz2",
 "unpack": true
+},
+{
+"algorithm": "sha512",
+"visibility": "public",
+"filename": "makecab.tar.bz2",
+"unpack": true,
+"digest": "da1f7685e5bc49a5ffbe5b4a3678ac7a58a0e125031726d30e2bacbbffa53d6efb9fd9f00d6dff5b54cff9412a103efa564c2af6f8fccc63082f6939181769f8",
+"size": 296777
 }
 ]
--- a/browser/config/tooltool-manifests/win32/releng.manifest
+++ b/browser/config/tooltool-manifests/win32/releng.manifest
@@ -24,10 +24,18 @@
 },
 {
 "version": "Visual Studio 2015 Update 3 14.0.25425.01 / SDK 10.0.14393.0",
 "size": 326656969,
 "digest": "babc414ffc0457d27f5a1ed24a8e4873afbe2f1c1a4075469a27c005e1babc3b2a788f643f825efedff95b79686664c67ec4340ed535487168a3482e68559bc7",
 "algorithm": "sha512",
 "filename": "vs2015u3.zip",
 "unpack": true
+},
+{
+"algorithm": "sha512",
+"visibility": "public",
+"filename": "makecab.tar.bz2",
+"unpack": true,
+"digest": "da1f7685e5bc49a5ffbe5b4a3678ac7a58a0e125031726d30e2bacbbffa53d6efb9fd9f00d6dff5b54cff9412a103efa564c2af6f8fccc63082f6939181769f8",
+"size": 296777
 }
 ]
--- a/browser/config/tooltool-manifests/win64/clang.manifest
+++ b/browser/config/tooltool-manifests/win64/clang.manifest
@@ -33,10 +33,18 @@
 },
 {
 "version": "clang 4.0pre/r286542",
 "size": 226755339,
 "digest": "3c598607c36e70788ca7dbdf0d835f9e44fbcaa7b1ed77ef9971d743a5a230bebc0ccd2bcdf97f63ed4546d1b83f4c3556f35c30589c755aaaefbd674f750e22",
 "algorithm": "sha512",
 "filename": "clang.tar.bz2",
 "unpack": true
+},
+{
+"algorithm": "sha512",
+"visibility": "public",
+"filename": "makecab.tar.bz2",
+"unpack": true,
+"digest": "da1f7685e5bc49a5ffbe5b4a3678ac7a58a0e125031726d30e2bacbbffa53d6efb9fd9f00d6dff5b54cff9412a103efa564c2af6f8fccc63082f6939181769f8",
+"size": 296777
 }
 ]
--- a/browser/config/tooltool-manifests/win64/releng.manifest
+++ b/browser/config/tooltool-manifests/win64/releng.manifest
@@ -25,10 +25,18 @@
 },
 {
 "version": "Visual Studio 2015 Update 3 14.0.25425.01 / SDK 10.0.14393.0",
 "size": 326656969,
 "digest": "babc414ffc0457d27f5a1ed24a8e4873afbe2f1c1a4075469a27c005e1babc3b2a788f643f825efedff95b79686664c67ec4340ed535487168a3482e68559bc7",
 "algorithm": "sha512",
 "filename": "vs2015u3.zip",
 "unpack": true
+},
+{
+"algorithm": "sha512",
+"visibility": "public",
+"filename": "makecab.tar.bz2",
+"unpack": true,
+"digest": "da1f7685e5bc49a5ffbe5b4a3678ac7a58a0e125031726d30e2bacbbffa53d6efb9fd9f00d6dff5b54cff9412a103efa564c2af6f8fccc63082f6939181769f8",
+"size": 296777
 }
 ]
--- a/build/moz.configure/windows.configure
+++ b/build/moz.configure/windows.configure
@@ -411,8 +411,11 @@ add_old_configure_assignment('LINK', lin
 @depends(sdk_bin_path)
 @imports('os')
 def alter_path(sdk_bin_path):
     path = os.pathsep.join(sdk_bin_path)
     os.environ['PATH'] = path
     return path
 
 set_config('PATH', alter_path)
+
+check_prog('MAKECAB', ('makecab.exe',))
+
--- a/build/mozconfig.win-common
+++ b/build/mozconfig.win-common
@@ -9,8 +9,9 @@ if [ "x$IS_NIGHTLY" = "xyes" ]; then
   MOZ_AUTOMATION_SDK=${MOZ_AUTOMATION_SDK-1}
 fi
 
 # Some builds (eg: Mulet) don't want the installer, so only set this if it
 # hasn't already been set.
 MOZ_AUTOMATION_INSTALLER=${MOZ_AUTOMATION_INSTALLER-1}
 
 export SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE=c:/builds/crash-stats-api.token
+export MAKECAB=$topsrcdir/makecab.exe
--- a/toolkit/crashreporter/tools/symbolstore.py
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -786,17 +786,18 @@ class Dumper_Win32(Dumper):
         # Cache the corrected version to avoid future filesystem hits.
         self.fixedFilenameCaseCache[file] = result
         return result
 
     def CopyDebug(self, file, debug_file, guid, code_file, code_id):
         def compress(path):
             compressed_file = path[:-1] + '_'
             # ignore makecab's output
-            success = subprocess.call(["makecab.exe", "/D",
+            makecab = buildconfig.substs['MAKECAB']
+            success = subprocess.call([makecab, "-D",
                                        "CompressionType=MSZIP",
                                        path, compressed_file],
                                       stdout=open(os.devnull, 'w'),
                                       stderr=subprocess.STDOUT)
             if success == 0 and os.path.exists(compressed_file):
                 os.unlink(path)
                 return True
             return False
--- a/toolkit/crashreporter/tools/unit-symbolstore.py
+++ b/toolkit/crashreporter/tools/unit-symbolstore.py
@@ -206,16 +206,17 @@ class TestCopyDebug(HelperMixin, unittes
                                                   symbol_path=self.symbol_dir,
                                                   copy_debug=True,
                                                   archs="abc xyz")
         d.CopyDebug = mock_copy_debug
         d.Process(self.test_dir)
         d.Finish(stop_pool=False)
         self.assertEqual(1, len(copied))
 
+    @patch.dict('buildconfig.substs', {'MAKECAB': 'makecab'})
     def test_copy_debug_copies_binaries(self):
         """
         Test that CopyDebug copies binaries as well on Windows.
         """
         test_file = os.path.join(self.test_dir, 'foo.pdb')
         write_pdb(test_file)
         code_file = 'foo.dll'
         code_id = 'abc123'