Bug 1208320 - Do not stage reftest test files before archiving; r=glandium
authorGregory Szorc <gps@mozilla.com>
Wed, 30 Sep 2015 17:39:33 -0700
changeset 265465 07a13667f8e0c4811170e24e1ee9f8d205ed0a7b
parent 265464 6416acd3ff9b35b0960f5c9d91fe8e571f440e4f
child 265466 c9611e5f49357042766c092e16cfb59cb2e8fbd2
push id29462
push usercbook@mozilla.com
push dateThu, 01 Oct 2015 10:39:47 +0000
treeherdermozilla-central@2c1fb007137d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1208320
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1208320 - Do not stage reftest test files before archiving; r=glandium This is slightly more involved than earlier changes because reftests have a one-off mechanism for finding files. Essentially, the master reftest manifest is loaded, directories are discovered, and every file in those directories is packaged. We add support to our test archive generation tool to read sources from reftest manifests and tell it where the reftest manifests are. print-manifest-dirs.py was only being used for staging reftest files. Since we don't do that any more, the functionality doesn't need to exist in a standalone file, so it has been moved inline into test_archive.py. This change avoids copying ~26,000 tests consuming 131 MB during test packaging. This is a majority of the file count that was remaining in the stage directory at this point. On my machine (which hasn't typically seen major wall time wins from not staging files due to its fast SSD), this change made test packaging ~20% faster, reducing wall time from ~50s to ~40s! A Try push seemed to indicate drastic results with the series up to this point. Including the already landed changes to generate test archives concurrently, test packaging times on OS X builders dropped from ~18:40 to 6:29! Times on Linux x64 remained about the same (~2:46). This is possibly due to these machines already having SSDs and due to normal variance in performance of builders and EC2 instances.
layout/tools/reftest/Makefile.in
layout/tools/reftest/print-manifest-dirs.py
python/mozbuild/mozbuild/action/test_archive.py
testing/testsuite-targets.mk
--- a/layout/tools/reftest/Makefile.in
+++ b/layout/tools/reftest/Makefile.in
@@ -43,20 +43,8 @@ include $(topsrcdir)/build/automation-bu
 $(_HARNESS_FILES): $(_DEST_DIR)
 
 # copy harness and the reftest extension bits to $(_DEST_DIR)
 # This needs to happen after jar.mn handling from rules.mk included above.
 # The order of the :: rules ensures that.
 libs:: $(_HARNESS_FILES) $(addprefix $(_DEST_DIR)/,$(_HARNESS_PP_FILES))
 	$(INSTALL) $(_HARNESS_FILES) $(_DEST_DIR)
 	(cd $(DIST)/xpi-stage && tar $(TAR_CREATE_FLAGS) - reftest) | (cd $(_DEST_DIR) && tar -xf -)
-
-PKG_STAGE = $(DIST)/test-stage
-
-# stage harness and tests for packaging
-stage-package:
-	$(NSINSTALL) -D $(PKG_STAGE)/reftest && $(NSINSTALL) -D $(PKG_STAGE)/reftest/tests
-	$(PYTHON) $(topsrcdir)/layout/tools/reftest/print-manifest-dirs.py \
-          $(topsrcdir) \
-          $(topsrcdir)/layout/reftests/reftest.list \
-          $(topsrcdir)/testing/crashtest/crashtests.list \
-          | (cd $(topsrcdir) && xargs tar $(TAR_CREATE_FLAGS) -) \
-          | (cd $(PKG_STAGE)/reftest/tests && tar -xf -)
deleted file mode 100644
--- a/layout/tools/reftest/print-manifest-dirs.py
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-import sys
-from reftest import ReftestManifest
-
-def printTestDirs(topsrcdir, topmanifests):
-    """Parse |topmanifests| and print a list of directories containing the tests
-    within (and the manifests including those tests), relative to |topsrcdir|.
-    """
-    topsrcdir = os.path.abspath(topsrcdir)
-    dirs = set()
-    for path in topmanifests:
-        m = ReftestManifest()
-        m.load(path)
-        dirs |= m.dirs
-
-    for d in sorted(dirs):
-        d = d[len(topsrcdir):].replace('\\', '/').lstrip('/')
-        print(d)
-
-if __name__ == '__main__':
-    if len(sys.argv) < 3:
-      print >>sys.stderr, "Usage: %s topsrcdir reftest.list [reftest.list]*" % sys.argv[0]
-      sys.exit(1)
-    printTestDirs(sys.argv[1], sys.argv[2:])
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@@ -93,21 +93,16 @@ ARCHIVE_FILES = {
         {
             'source': buildconfig.topsrcdir,
             'base': 'testing',
             'pattern': 'mozharness/**',
         },
     ],
     'reftest': [
         {
-            'source': STAGE,
-            'base': '',
-            'pattern': 'reftest/**',
-        },
-        {
             'source': buildconfig.topobjdir,
             'base': '_tests',
             'pattern': 'reftest/**',
         },
         {
             'source': buildconfig.topobjdir,
             'base': '',
             'pattern': 'mozinfo.json',
@@ -163,18 +158,18 @@ for k, v in ARCHIVE_FILES.items():
 
     if not any(p.startswith('%s/' % k) for p in ignores):
         raise Exception('"common" ignore list probably should contain %s' % k)
 
 
 def find_files(archive):
     for entry in ARCHIVE_FILES[archive]:
         source = entry['source']
-        base = entry['base']
-        pattern = entry['pattern']
+        base = entry.get('base', '')
+        pattern = entry.get('pattern')
         dest = entry.get('dest')
         ignore = list(entry.get('ignore', []))
         ignore.append('**/.mkdir.done')
         ignore.append('**/*.pyc')
 
         common_kwargs = {
             'find_executables': False,
             'find_dotfiles': True,
@@ -184,27 +179,81 @@ def find_files(archive):
         finder = FileFinder(os.path.join(source, base), **common_kwargs)
 
         for p, f in finder.find(pattern):
             if dest:
                 p = mozpath.join(dest, p)
             yield p, f
 
 
+def find_reftest_dirs(topsrcdir, manifests):
+    from reftest import ReftestManifest
+
+    dirs = set()
+    for p in manifests:
+        m = ReftestManifest()
+        m.load(os.path.join(topsrcdir, p))
+        dirs |= m.dirs
+
+    dirs = {mozpath.normpath(d[len(topsrcdir):]).lstrip('/') for d in dirs}
+
+    # Filter out children captured by parent directories because duplicates
+    # will confuse things later on.
+    def parents(p):
+        while True:
+            p = mozpath.dirname(p)
+            if not p:
+                break
+            yield p
+
+    seen = set()
+    for d in sorted(dirs, key=len):
+        if not any(p in seen for p in parents(d)):
+            seen.add(d)
+
+    return sorted(seen)
+
+
+def insert_reftest_entries(entries):
+    """Reftests have their own mechanism for defining tests and locations.
+
+    This function is called when processing the reftest archive to process
+    reftest test manifests and insert the results into the existing list of
+    archive entries.
+    """
+    manifests = (
+        'layout/reftests/reftest.list',
+        'testing/crashtest/crashtests.list',
+    )
+
+    for base in find_reftest_dirs(buildconfig.topsrcdir, manifests):
+        entries.append({
+            'source': buildconfig.topsrcdir,
+            'base': '',
+            'pattern': '%s/**' % base,
+            'dest': 'reftest/tests',
+        })
+
+
 def main(argv):
     parser = argparse.ArgumentParser(
         description='Produce test archives')
     parser.add_argument('archive', help='Which archive to generate')
     parser.add_argument('outputfile', help='File to write output to')
 
     args = parser.parse_args(argv)
 
     if not args.outputfile.endswith('.zip'):
         raise Exception('expected zip output file')
 
+    # Adjust reftest entries only if processing reftests (because it is
+    # unnecessary overhead otherwise).
+    if args.archive == 'reftest':
+        insert_reftest_entries(ARCHIVE_FILES['reftest'])
+
     with open(args.outputfile, 'wb') as fh:
         with JarWriter(fileobj=fh, optimize=False) as writer:
             res = find_files(args.archive)
             for p, f in res:
                 writer.add(p.encode('utf-8'), f.read(), mode=f.mode)
 
 
 if __name__ == '__main__':
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -369,17 +369,16 @@ pgo-profile-run:
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 PKG_STAGE = $(DIST)/test-stage
 
 stage-all: \
   stage-config \
   stage-mach \
   stage-mochitest \
-  stage-reftest \
   stage-xpcshell \
   stage-jstests \
   stage-jetpack \
   stage-mozbase \
   stage-tps \
   stage-modules \
   stage-marionette \
   stage-cppunittests \
@@ -480,19 +479,16 @@ stage-mach: make-stage-dir
 	cp $(topsrcdir)/mach $(PKG_STAGE)
 
 stage-mochitest: make-stage-dir
 	$(MAKE) -C $(DEPTH)/testing/mochitest stage-package
 ifeq ($(MOZ_BUILD_APP),mobile/android)
 	$(NSINSTALL) $(DEPTH)/mobile/android/base/fennec_ids.txt $(PKG_STAGE)/mochitest
 endif
 
-stage-reftest: make-stage-dir
-	$(MAKE) -C $(DEPTH)/layout/tools/reftest stage-package
-
 stage-xpcshell: make-stage-dir
 	$(MAKE) -C $(DEPTH)/testing/xpcshell stage-package
 
 stage-jstests: make-stage-dir
 	$(MAKE) -C $(DEPTH)/js/src/tests stage-package
 
 stage-android: make-stage-dir
 ifdef MOZ_ENABLE_SZIP
@@ -596,17 +592,16 @@ stage-instrumentation-tests: make-stage-
   package-tests-prepare-dest \
   package-tests-mozharness \
   package-tests-common \
   make-stage-dir \
   stage-all \
   stage-b2g \
   stage-config \
   stage-mochitest \
-  stage-reftest \
   stage-xpcshell \
   stage-jstests \
   stage-android \
   stage-jetpack \
   stage-mozbase \
   stage-tps \
   stage-modules \
   stage-marionette \