Bug 1359965 - Support and generate tar.gz WPT archive; r=glandium
authorGregory Szorc <gps@mozilla.com>
Mon, 08 May 2017 17:19:05 -0700
changeset 410037 11ba6213be5a745d70e5c6fdcd036bafb1ac2718
parent 410036 0a94b9164a649af95e7e645669bf6b2a1883745c
child 410038 2b5bcdc6c93b558108654815201c14774f6218c1
push id7391
push usermtabara@mozilla.com
push dateMon, 12 Jun 2017 13:08:53 +0000
treeherdermozilla-beta@2191d7f87e2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1359965
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1359965 - Support and generate tar.gz WPT archive; r=glandium Several years ago there was a single zip file for all test files. Clients would only extract the files they needed. Thus, zip was a reasonable archive format because it allowed direct access to members without having to decompress the entirety of the stream. We have since split up that monolithic archive into separate, domain-specific archives. e.g. 1 archive for mochitests and one for xpcshell tests. This drastically cut down on network I/O required on testers because they only fetched archives/data that was relevant. It also enabled parallel generation of test archives, we shaved dozens of seconds off builds due to compression being a long pole. Despite the architectural changes to test archive management, we still used zip files. This is not ideal because we no longer access specific files in test archives and thus don't care about single/partial member access performance. This commit implements support for generating tar.gz test archives. And it switches the web-platform archive to a tar.gz file. The performance implications for archive generation are significant: before: 48,321,250 bytes; 6.05s after: 31,844,267 bytes; 4.57s The size is reduced because we have a single compression context so data from 1 file can benefit compression in a subsequent file. CPU usage is reduced because the compressor has to work less with 1 context than it does with N. While I didn't measure it, decompression performance should also be improved for the same reasons. And of course network I/O will be reduced. mozharness consumers use a generic method for handling unarchiving. This method automagically handles multiple file extensions. So as long as downstream consumers aren't hard coding ".zip" this change should "just work." MozReview-Commit-ID: LQa5MIHLsms
python/mozbuild/mozbuild/action/test_archive.py
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
testing/testsuite-targets.mk
toolkit/mozapps/installer/package-name.mk
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@@ -15,16 +15,17 @@ import itertools
 import os
 import sys
 import time
 
 from manifestparser import TestManifest
 from reftest import ReftestManifest
 
 from mozbuild.util import ensureParentDir
+from mozpack.archive import create_tar_gz_from_files
 from mozpack.copier import FileRegistry
 from mozpack.files import ExistingFile, FileFinder
 from mozpack.manifests import InstallManifest
 from mozpack.mozjar import JarWriter
 import mozpack.path as mozpath
 
 import buildconfig
 
@@ -609,32 +610,41 @@ def find_manifest_dirs(topsrcdir, manife
 def main(argv):
     parser = argparse.ArgumentParser(
         description='Produce test archives')
     parser.add_argument('archive', help='Which archive to generate')
     parser.add_argument('outputfile', help='File to write output to')
 
     args = parser.parse_args(argv)
 
-    if not args.outputfile.endswith('.zip'):
-        raise Exception('expected zip output file')
+    out_file = args.outputfile
+    if not out_file.endswith(('.tar.gz', '.zip')):
+        raise Exception('expected tar.gz or zip output file')
 
     file_count = 0
     t_start = time.time()
-    ensureParentDir(args.outputfile)
-    with open(args.outputfile, 'wb') as fh:
+    ensureParentDir(out_file)
+    res = find_files(args.archive)
+    with open(out_file, 'wb') as fh:
         # Experimentation revealed that level 5 is significantly faster and has
         # marginally larger sizes than higher values and is the sweet spot
         # for optimal compression. Read the detailed commit message that
         # introduced this for raw numbers.
-        with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
-            res = find_files(args.archive)
-            for p, f in res:
-                writer.add(p.encode('utf-8'), f.read(), mode=f.mode, skip_duplicates=True)
-                file_count += 1
+        if out_file.endswith('.tar.gz'):
+            files = dict(res)
+            create_tar_gz_from_files(fh, files, compresslevel=5)
+            file_count = len(files)
+        elif out_file.endswith('.zip'):
+            with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
+                for p, f in res:
+                    writer.add(p.encode('utf-8'), f.read(), mode=f.mode,
+                               skip_duplicates=True)
+                    file_count += 1
+        else:
+            raise Exception('unhandled file extension: %s' % out_file)
 
     duration = time.time() - t_start
     zip_size = os.path.getsize(args.outputfile)
     basename = os.path.basename(args.outputfile)
     print('Wrote %d files in %d bytes to %s in %.2fs' % (
           file_count, zip_size, basename, duration))
 
 
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -27,17 +27,17 @@ from voluptuous import Any, Required, Op
     "target.json",
     "target.mochitest.tests.zip",
     "target.mozinfo.json",
     "target.reftest.tests.zip",
     "target.talos.tests.zip",
     "target.awsy.tests.zip",
     "target.test_packages.json",
     "target.txt",
-    "target.web-platform.tests.zip",
+    "target.web-platform.tests.tar.gz",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "target.jsshell.zip",
     "mozharness.zip",
     "target.langpack.xpi",
     "host/bin/mar",
     "host/bin/mbsdiff",
 ]
@@ -74,17 +74,17 @@ from voluptuous import Any, Required, Op
     "en-US/target.json",
     "en-US/target.mochitest.tests.zip",
     "en-US/target.mozinfo.json",
     "en-US/target.reftest.tests.zip",
     "en-US/target.talos.tests.zip",
     "en-US/target.awsy.tests.zip",
     "en-US/target.test_packages.json",
     "en-US/target.txt",
-    "en-US/target.web-platform.tests.zip",
+    "en-US/target.web-platform.tests.tar.gz",
     "en-US/target.xpcshell.tests.zip",
     "en-US/target_info.txt",
     "en-US/bouncer.apk",
     "en-US/mozharness.zip",
     "en-US/robocop.apk",
     "en-US/target.jsshell.zip",
 ]
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
@@ -98,17 +98,17 @@ from voluptuous import Any, Required, Op
     "target.json",
     "target.mochitest.tests.zip",
     "target.mozinfo.json",
     "target.reftest.tests.zip",
     "target.talos.tests.zip",
     "target.awsy.tests.zip",
     "target.test_packages.json",
     "target.txt",
-    "target.web-platform.tests.zip",
+    "target.web-platform.tests.tar.gz",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "bouncer.apk",
     "mozharness.zip",
     "robocop.apk",
     "target.jsshell.zip",
 ]
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -29,17 +29,17 @@ logger = logging.getLogger(__name__)
     "target.json",
     "target.mochitest.tests.zip",
     "target.mozinfo.json",
     "target.reftest.tests.zip",
     "target.talos.tests.zip",
     "target.awsy.tests.zip",
     "target.test_packages.json",
     "target.txt",
-    "target.web-platform.tests.zip",
+    "target.web-platform.tests.tar.gz",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "target.jsshell.zip",
     "mozharness.zip",
     "target.langpack.xpi",
     "host/bin/mar",
     "host/bin/mbsdiff",
 ]
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -114,56 +114,61 @@ stage-all: \
 ifdef MOZ_WEBRTC
 stage-all: stage-steeplechase
 endif
 
 ifdef COMPILE_ENVIRONMENT
 stage-all: stage-cppunittests
 endif
 
-TEST_PKGS := \
+TEST_PKGS_ZIP := \
   common \
   cppunittest \
   mochitest \
   reftest \
   talos \
   awsy \
+  xpcshell \
+  $(NULL)
+
+TEST_PKGS_TARGZ := \
   web-platform \
-  xpcshell \
   $(NULL)
 
 ifdef LINK_GTEST_DURING_COMPILE
 stage-all: stage-gtest
-TEST_PKGS += gtest
+TEST_PKGS_ZIP += gtest
 endif
 
-PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.zip'
+PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.$(2)'
 
 test-packages-manifest:
 	@rm -f $(MOZ_TEST_PACKAGES_FILE)
 	$(NSINSTALL) -D $(dir $(MOZ_TEST_PACKAGES_FILE))
 	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
       --jsshell $(JSSHELL_NAME) \
       --dest-file '$(MOZ_TEST_PACKAGES_FILE)' \
-      $(call PKG_ARG,common) \
-      $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
+      $(call PKG_ARG,common,zip) \
+      $(foreach pkg,$(TEST_PKGS_ZIP),$(call PKG_ARG,$(pkg),zip)) \
+      $(foreach pkg,$(TEST_PKGS_TARGZ),$(call PKG_ARG,$(pkg),tar.gz))
 
 package-tests-prepare-dest:
 	@rm -f '$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)'
 	$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
 
 define package_archive
 package-tests-$(1): stage-all package-tests-prepare-dest
 	$$(call py_action,test_archive, \
 		$(1) \
-		'$$(abspath $$(DIST))/$$(PKG_PATH)/$$(PKG_BASENAME).$(1).tests.zip')
+		'$$(abspath $$(DIST))/$$(PKG_PATH)/$$(PKG_BASENAME).$(1).tests.$(2)')
 package-tests: package-tests-$(1)
 endef
 
-$(foreach name,$(TEST_PKGS),$(eval $(call package_archive,$(name))))
+$(foreach name,$(TEST_PKGS_ZIP),$(eval $(call package_archive,$(name),zip)))
+$(foreach name,$(TEST_PKGS_TARGZ),$(eval $(call package_archive,$(name),tar.gz)))
 
 ifeq ($(MOZ_BUILD_APP),mobile/android)
 stage-all: stage-android
 stage-all: stage-instrumentation-tests
 endif
 
 # Prepare _tests before any of the other staging/packaging steps.
 # make-stage-dir is a prerequisite to all the stage-* targets in testsuite-targets.mk.
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -87,17 +87,17 @@ CODE_COVERAGE_ARCHIVE_BASENAME = $(PKG_B
 MOZHARNESS_PACKAGE = mozharness.zip
 
 # Test package naming
 TEST_PACKAGE = $(PKG_BASENAME).common.tests.zip
 CPP_TEST_PACKAGE = $(PKG_BASENAME).cppunittest.tests.zip
 XPC_TEST_PACKAGE = $(PKG_BASENAME).xpcshell.tests.zip
 MOCHITEST_PACKAGE = $(PKG_BASENAME).mochitest.tests.zip
 REFTEST_PACKAGE = $(PKG_BASENAME).reftest.tests.zip
-WP_TEST_PACKAGE = $(PKG_BASENAME).web-platform.tests.zip
+WP_TEST_PACKAGE = $(PKG_BASENAME).web-platform.tests.tar.gz
 TALOS_PACKAGE = $(PKG_BASENAME).talos.tests.zip
 AWSY_PACKAGE = $(PKG_BASENAME).awsy.tests.zip
 GTEST_PACKAGE = $(PKG_BASENAME).gtest.tests.zip
 
 ifneq (,$(wildcard $(DIST)/bin/application.ini))
 BUILDID = $(shell $(PYTHON) $(MOZILLA_DIR)/config/printconfigsetting.py $(DIST)/bin/application.ini App BuildID)
 else
 BUILDID = $(shell $(PYTHON) $(MOZILLA_DIR)/config/printconfigsetting.py $(DIST)/bin/platform.ini Build BuildID)