Bug 917999 - Part 1 - Write out and upload a manifest of test archives and the harnesses that depend on them.;r=ted
authorChris Manchester <cmanchester@mozilla.com>
Fri, 15 May 2015 11:30:24 -0700
changeset 244121 8deed4a4ccded653fcce7e9df3dcfa0c54ed1ab8
parent 244120 b3902a6225c3d585976f6c3830a692155f9cce46
child 244122 acf9c5ff8fdca1676e08452343789d4d7db82d52
push id12987
push userphilringnalda@gmail.com
push dateSat, 16 May 2015 16:53:11 +0000
treeherderfx-team@2f6ea66057fe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs917999
milestone41.0a1
Bug 917999 - Part 1 - Write out and upload a manifest of test archives and the harnesses that depend on them.;r=ted
b2g/config/emulator-ics/config.json
b2g/config/emulator/config.json
build/gen_mach_buildprops.py
build/gen_test_packages_manifest.py
testing/testsuite-targets.mk
toolkit/mozapps/installer/package-name.mk
toolkit/mozapps/installer/upload-files.mk
--- a/b2g/config/emulator-ics/config.json
+++ b/b2g/config/emulator-ics/config.json
@@ -5,16 +5,17 @@
     "mock_packages": ["ccache", "make", "bison", "flex", "gcc", "g++", "mpfr", "zlib-devel", "ncurses-devel", "zip", "autoconf213", "glibc-static", "perl-Digest-SHA", "wget", "alsa-lib", "atk", "cairo", "dbus-glib", "fontconfig", "freetype", "glib2", "gtk2", "libXRender", "libXt", "pango", "mozilla-python27-mercurial", "openssh-clients", "nss-devel", "glibc-devel.i686", "libstdc++.i686", "zlib-devel.i686", "ncurses-devel.i686", "libX11-devel.i686", "mesa-libGL-devel.i686", "mesa-libGL-devel", "libX11-devel", "git", "libxml2"],
     "mock_files": [["/home/cltbld/.ssh", "/home/mock_mozilla/.ssh"]],
     "build_targets": ["droid", "package-emulator", "package-tests"],
     "upload_files": [
         "{workdir}/out/target/product/generic/*.tar.bz2",
         "{workdir}/out/target/product/generic/tests/*.zip",
         "{workdir}/out/emulator.tar.gz",
         "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
+        "{objdir}/dist/test_packages.json",
         "{workdir}/sources.xml"
     ],
     "upload_platform": "emulator-ics",
     "gecko_l10n_root": "https://hg.mozilla.org/l10n-central",
     "gaia": {
         "l10n": {
             "vcs": "hgtool",
             "root": "https://hg.mozilla.org/gaia-l10n"
--- a/b2g/config/emulator/config.json
+++ b/b2g/config/emulator/config.json
@@ -5,16 +5,17 @@
     "mock_packages": ["ccache", "make", "bison", "flex", "gcc", "g++", "mpfr", "zlib-devel", "ncurses-devel", "zip", "autoconf213", "glibc-static", "perl-Digest-SHA", "wget", "alsa-lib", "atk", "cairo", "dbus-glib", "fontconfig", "freetype", "glib2", "gtk2", "libXRender", "libXt", "pango", "mozilla-python27-mercurial", "openssh-clients", "nss-devel", "glibc-devel.i686", "libstdc++.i686", "zlib-devel.i686", "ncurses-devel.i686", "libX11-devel.i686", "mesa-libGL-devel.i686", "mesa-libGL-devel", "libX11-devel", "git", "libxml2"],
     "mock_files": [["/home/cltbld/.ssh", "/home/mock_mozilla/.ssh"]],
     "build_targets": ["droid", "package-emulator", "package-tests"],
     "upload_files": [
         "{workdir}/out/target/product/generic/*.tar.bz2",
         "{workdir}/out/target/product/generic/tests/*.zip",
         "{workdir}/out/emulator.tar.gz",
         "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
+        "{objdir}/dist/test_packages.json",
         "{workdir}/sources.xml"
     ],
     "public_upload_files": [
         "{workdir}/out/target/product/generic/*.tar.bz2",
         "{workdir}/out/target/product/generic/tests/*.zip",
         "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
         "{objdir}/dist/b2g-*.tar.gz",
         "{workdir}/sources.xml"
--- a/build/gen_mach_buildprops.py
+++ b/build/gen_mach_buildprops.py
@@ -51,20 +51,22 @@ def getUrlProperties(filename):
         ('unsignedApkUrl', lambda m: m.endswith('apk') and
                            'unsigned-unaligned' in m),
         ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m),
         ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
         ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
         ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m),
         ('codeCoverageURL', lambda m: m.endswith('code-coverage-gcno.zip')),
         ('sdkUrl', lambda m: m.endswith(('sdk.tar.bz2', 'sdk.zip'))),
+        ('testPackagesUrl', lambda m: m.endswith('test_packages.json')),
         # packageUrl must be last!
-        ('packageUrl', lambda m: True),
+        ('packageUrl', lambda m: (not m.endswith('.json') and
+                                  not m.endswith('tests.zip'))),
     ]
-    url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''')
+    url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz|json))$''')
     properties = {}
 
     try:
         with open(filename) as f:
             for line in f:
                 m = url_re.match(line)
                 if m:
                     m = m.group(1)
new file mode 100644
--- /dev/null
+++ b/build/gen_test_packages_manifest.py
@@ -0,0 +1,63 @@
+#!/usr/bin/python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+
+from argparse import ArgumentParser
+
+ALL_HARNESSES = [
+    'common', # Harnesses without a specific package will look here.
+    'mochitest',
+    'reftest',
+    'webapprt',
+    'xpcshell',
+    'cppunittest',
+    'jittest',
+    'mozbase',
+]
+
+PACKAGE_SPECIFIED_HARNESSES = [
+]
+
+
+def parse_args():
+    parser = ArgumentParser(description='Generate a test_packages.json file to tell automation which harnesses require which test packages.')
+    parser.add_argument("--common", required=True,
+                        action="store", dest="tests_common",
+                        help="Name of the \"common\" archive, a package to be used by all harnesses.")
+    parser.add_argument("--jsshell", required=True,
+                        action="store", dest="jsshell",
+                        help="Name of the jsshell zip.")
+    for harness in PACKAGE_SPECIFIED_HARNESSES:
+        parser.add_argument("--%s" % harness, required=True,
+                            action="store", dest=harness,
+                            help="Name of the %s zip." % harness)
+    parser.add_argument("--dest-file", required=True,
+                        action="store", dest="destfile",
+                        help="Path to the output file to be written.")
+    return parser.parse_args()
+
+def generate_package_data(args):
+    # Generate a dictionary mapping test harness names (exactly as they're known to
+    # mozharness and testsuite-targets.mk, ideally) to the set of archive names that
+    # harness depends on to run.
+    # mozharness will use this file to determine what test zips to download,
+    # which will be an optimization once parts of the main zip are split to harness
+    # specific zips.
+    tests_common = args.tests_common
+    jsshell = args.jsshell
+
+    harness_requirements = dict([(k, [tests_common]) for k in ALL_HARNESSES])
+    harness_requirements['jittest'].append(jsshell)
+    for harness in PACKAGE_SPECIFIED_HARNESSES:
+        harness_requirements[harness].append(getattr(args, harness))
+    return harness_requirements
+
+if __name__ == '__main__':
+    args = parse_args()
+    packages_data = generate_package_data(args)
+    with open(args.destfile, 'w') as of:
+        json.dump(packages_data, of, indent=4)
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -401,25 +401,30 @@ package-tests: \
   stage-mozbase \
   stage-tps \
   stage-modules \
   stage-marionette \
   stage-cppunittests \
   stage-jittest \
   stage-web-platform-tests \
   stage-luciddream \
+  test-packages-manifest \
   $(NULL)
 ifdef MOZ_WEBRTC
 package-tests: stage-steeplechase
 endif
 else
 # This staging area has been built for us by universal/flight.mk
 PKG_STAGE = $(DIST)/universal/test-stage
 endif
 
+test-packages-manifest:
+	@rm -f $(MOZ_TEST_PACKAGES_FILE)
+	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py --common $(TEST_PACKAGE) --jsshell $(JSSHELL_NAME) --dest-file $(MOZ_TEST_PACKAGES_FILE)
+
 package-tests:
 	@rm -f '$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)'
 ifndef UNIVERSAL_BINARY
 	$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
 endif
 	$(MKDIR) -p $(abspath $(DIST))/$(PKG_PATH) && \
 	cd $(PKG_STAGE) && \
 	  zip -rq9D '$(abspath $(DIST))/$(PKG_PATH)$(TEST_PACKAGE)' \
@@ -589,10 +594,11 @@ stage-instrumentation-tests: make-stage-
   stage-mozbase \
   stage-tps \
   stage-modules \
   stage-marionette \
   stage-steeplechase \
   stage-web-platform-tests \
   stage-instrumentation-tests \
   stage-luciddream \
+  test-packages-manifest \
   $(NULL)
 
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -150,13 +150,15 @@ MOZ_SOURCE_STAMP = $(firstword $(shell h
 ###########################################################################
 ifdef MOZ_INCLUDE_SOURCE_INFO
 MOZ_SOURCE_REPO = $(call getSourceRepo,$(MOZILLA_DIR)$(NULL) $(NULL))
 endif
 
 MOZ_SOURCESTAMP_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).txt
 MOZ_BUILDINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).json
 MOZ_MOZINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).mozinfo.json
+MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/test_packages.json
 
 # JavaScript Shell
-PKG_JSSHELL = $(DIST)/jsshell-$(MOZ_PKG_PLATFORM).zip
+JSSHELL_NAME = jsshell-$(MOZ_PKG_PLATFORM).zip
+PKG_JSSHELL = $(DIST)/$(JSSHELL_NAME)
 
 endif # PACKAGE_NAME_MK_INCLUDED
--- a/toolkit/mozapps/installer/upload-files.mk
+++ b/toolkit/mozapps/installer/upload-files.mk
@@ -721,16 +721,17 @@ UPLOAD_FILES= \
   $(call QUOTED_WILDCARD,$(DIST)/$(LANGPACK)) \
   $(call QUOTED_WILDCARD,$(wildcard $(DIST)/$(PARTIAL_MAR))) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip) \
   $(call QUOTED_WILDCARD,$(DIST)/$(SDK)) \
   $(call QUOTED_WILDCARD,$(MOZ_SOURCESTAMP_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_BUILDINFO_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_MOZINFO_FILE)) \
+  $(call QUOTED_WILDCARD,$(MOZ_TEST_PACKAGES_FILE)) \
   $(call QUOTED_WILDCARD,$(PKG_JSSHELL)) \
   $(if $(UPLOAD_EXTRA_FILES), $(foreach f, $(UPLOAD_EXTRA_FILES), $(wildcard $(DIST)/$(f))))
 
 ifdef MOZ_CRASHREPORTER_UPLOAD_FULL_SYMBOLS
 UPLOAD_FILES += \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip)
 endif