Bug 1208320 - Produce xpcshell archive without staging test files; r=glandium
authorGregory Szorc <gps@mozilla.com>
Wed, 30 Sep 2015 09:36:16 -0700
changeset 265457 1c9cf4ccedaaa1a237ae8f7a46bf6ea297442efc
parent 265456 3c91cbd21682ef1c095535d353af795e22a378fa
child 265458 5545643ec0424036732c21056b8afce2ac24c1cf
push id29462
push usercbook@mozilla.com
push dateThu, 01 Oct 2015 10:39:47 +0000
treeherdermozilla-central@2c1fb007137d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1208320
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1208320 - Produce xpcshell archive without staging test files; r=glandium This commit produces the xpcshell test archive without staging 5000+ xpcshell test files first. We teach the archiver to ignore .mkdir.done files. The xpcshell Makefile.in still stages some files. This is less than ideal. However, it is a small handful of files and shouldn't add too much overhead. This appears to not impact overall CPU usage significantly on my machine, despute using Python instead of `zip`. It does reduce I/O by ~25MB by avoiding the staging copy.
python/mozbuild/mozbuild/action/test_archive.py
testing/testsuite-targets.mk
testing/xpcshell/Makefile.in
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@@ -11,56 +11,83 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import os
 import sys
 
 from mozpack.files import FileFinder
 from mozpack.mozjar import JarWriter
+import mozpack.path as mozpath
 
 import buildconfig
 
+STAGE = mozpath.join(buildconfig.topobjdir, 'dist', 'test-stage')
+
 
 ARCHIVE_FILES = {
     'mozharness': [
         {
             'source': buildconfig.topsrcdir,
             'base': 'testing',
             'pattern': 'mozharness/**',
         },
     ],
+    'xpcshell': [
+        {
+            'source': buildconfig.topobjdir,
+            'base': '_tests/xpcshell',
+            'pattern': '**',
+            'dest': 'xpcshell/tests',
+        },
+        {
+            'source': STAGE,
+            'base': '',
+            'pattern': 'xpcshell/**',
+        },
+    ],
 }
 
 
 def find_files(archive):
     for entry in ARCHIVE_FILES[archive]:
         source = entry['source']
         base = entry['base']
         pattern = entry['pattern']
+        dest = entry.get('dest')
+        ignore = list(entry.get('ignore', []))
+        ignore.append('**/.mkdir.done')
+        ignore.append('**/*.pyc')
 
-        finder = FileFinder(os.path.join(source, base),
-                            find_executables=False,
-                            find_dotfiles=True)
+        common_kwargs = {
+            'find_executables': False,
+            'find_dotfiles': True,
+            'ignore': ignore,
+        }
 
-        for f in finder.find(pattern):
-            yield f
+        finder = FileFinder(os.path.join(source, base), **common_kwargs)
+
+        for p, f in finder.find(pattern):
+            if dest:
+                p = mozpath.join(dest, p)
+            yield p, f
 
 
 def main(argv):
     parser = argparse.ArgumentParser(
         description='Produce test archives')
     parser.add_argument('archive', help='Which archive to generate')
     parser.add_argument('outputfile', help='File to write output to')
 
     args = parser.parse_args(argv)
 
     if not args.outputfile.endswith('.zip'):
         raise Exception('expected zip output file')
 
     with open(args.outputfile, 'wb') as fh:
         with JarWriter(fileobj=fh, optimize=False) as writer:
-            for p, f in find_files(args.archive):
+            res = find_files(args.archive)
+            for p, f in res:
                 writer.add(p.encode('utf-8'), f.read(), mode=f.mode)
 
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv[1:]))
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -394,67 +394,84 @@ ifdef MOZ_WEBRTC
 stage-all: stage-steeplechase
 endif
 
 TEST_PKGS := \
   cppunittest \
   mochitest \
   reftest \
   talos \
-  xpcshell \
   web-platform \
   $(NULL)
 
+PYTHON_TEST_PKGS := \
+  xpcshell \
+  $(NULL)
+
+ALL_TEST_PKGS := $(TEST_PKGS) $(PYTHON_TEST_PKGS)
+
 PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.zip'
 
 test-packages-manifest-tc:
 	@rm -f $(MOZ_TEST_PACKAGES_FILE_TC)
 	$(NSINSTALL) -D $(dir $(MOZ_TEST_PACKAGES_FILE_TC))
 	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
       --jsshell $(JSSHELL_NAME) \
       --dest-file $(MOZ_TEST_PACKAGES_FILE_TC) \
       --use-short-names \
       $(call PKG_ARG,common) \
-      $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
+      $(foreach pkg,$(ALL_TEST_PKGS),$(call PKG_ARG,$(pkg)))
 
 test-packages-manifest:
 	@rm -f $(MOZ_TEST_PACKAGES_FILE)
 	$(NSINSTALL) -D $(dir $(MOZ_TEST_PACKAGES_FILE))
 	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
       --jsshell $(JSSHELL_NAME) \
       --dest-file $(MOZ_TEST_PACKAGES_FILE) \
       $(call PKG_ARG,common) \
-      $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
+      $(foreach pkg,$(ALL_TEST_PKGS),$(call PKG_ARG,$(pkg)))
 
 package-tests-prepare-dest:
 	@rm -f '$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)'
 	$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
 
 package-tests-mozharness: package-tests-prepare-dest
-	$(call py_action,test_archive,mozharness $(abspath $(DIST))/$(PKG_PATH)/mozharness.zip)
+	$(call py_action,test_archive, \
+		mozharness \
+		$(abspath $(DIST))/$(PKG_PATH)/mozharness.zip)
 package-tests: package-tests-mozharness
 
 package-tests-common: stage-all package-tests-prepare-dest
 	cd $(abspath $(PKG_STAGE)) && \
 	  zip -rq9D '$(abspath $(DIST))/$(PKG_PATH)$(TEST_PACKAGE)' \
-	  * -x \*/.mkdir.done \*.pyc $(foreach name,$(TEST_PKGS),$(name)\*)
+	  * -x \*/.mkdir.done \*.pyc $(foreach name,$(ALL_TEST_PKGS),$(name)\*)
 package-tests: package-tests-common
 
 define package_archive
 package-tests-$(1): stage-all package-tests-prepare-dest
 	rm -f '$$(DIST)/$$(PKG_PATH)$$(PKG_BASENAME).$(1).tests.zip' && \
 		cd $$(abspath $(PKG_STAGE)) && \
 		zip -rq9D '$$(abspath $$(DIST))/$$(PKG_PATH)$$(PKG_BASENAME).$(1).tests.zip' \
 		$(1) -x \*/.mkdir.done \*.pyc ;
 .PHONY += package-tests-$(1)
 package-tests: package-tests-$(1)
 endef
 
 $(foreach name,$(TEST_PKGS),$(eval $(call package_archive,$(name))))
 
+define python_test_archive
+package-tests-$(1): stage-all package-tests-prepare-dest
+	$$(call py_action,test_archive, \
+		$(1) \
+		$$(abspath $$(DIST))/$$(PKG_PATH)/$$(PKG_BASENAME).$(1).tests.zip)
+package-tests: package-tests-$(1)
+endef
+
+$(foreach name,$(PYTHON_TEST_PKGS),$(eval $(call python_test_archive,$(name))))
+
 ifeq ($(MOZ_BUILD_APP),mobile/android)
 stage-all: stage-android
 stage-all: stage-instrumentation-tests
 endif
 
 ifeq ($(MOZ_BUILD_APP),mobile/android/b2gdroid)
 stage-all: stage-android
 endif
--- a/testing/xpcshell/Makefile.in
+++ b/testing/xpcshell/Makefile.in
@@ -37,10 +37,9 @@ libs::
 	cp $(DEPTH)/_tests/xpcshell/xpcshell.ini $(DEPTH)/_tests/xpcshell/all-test-dirs.list
 
 stage-package:
 	$(NSINSTALL) -D $(PKG_STAGE)/xpcshell/tests
 	@(cd $(srcdir) && tar $(TAR_CREATE_FLAGS) - $(TEST_HARNESS_FILES)) | (cd $(PKG_STAGE)/xpcshell && tar -xf -)
 	@(cd $(topsrcdir)/build && tar $(TAR_CREATE_FLAGS) - $(EXTRA_BUILD_FILES)) | (cd $(PKG_STAGE)/xpcshell && tar -xf -)
 	@cp $(DEPTH)/mozinfo.json $(PKG_STAGE)/xpcshell
 	@cp $(DEPTH)/build/automation.py $(PKG_STAGE)/xpcshell
-	(cd $(DEPTH)/_tests/xpcshell/ && tar $(TAR_CREATE_FLAGS) - *) | (cd $(PKG_STAGE)/xpcshell/tests && tar -xf -)
 	@(cd $(DIST)/bin/components && tar $(TAR_CREATE_FLAGS) - $(TEST_HARNESS_COMPONENTS)) | (cd $(PKG_STAGE)/bin/components && tar -xf -)