Bug 917999 - Part 1 - Write out and upload a manifest of test archives and the harnesses that depend on them.;r=ted
authorChris Manchester <cmanchester@mozilla.com>
Fri, 15 May 2015 11:30:24 -0700
changeset 244063 8deed4a4ccded653fcce7e9df3dcfa0c54ed1ab8
parent 244062 b3902a6225c3d585976f6c3830a692155f9cce46
child 244064 acf9c5ff8fdca1676e08452343789d4d7db82d52
push id59829
push usercmanchester@mozilla.com
push dateFri, 15 May 2015 18:30:32 +0000
treeherdermozilla-inbound@8deed4a4ccde [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs917999
milestone41.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 917999 - Part 1 - Write out and upload a manifest of test archives and the harnesses that depend on them.;r=ted
b2g/config/emulator-ics/config.json
b2g/config/emulator/config.json
build/gen_mach_buildprops.py
build/gen_test_packages_manifest.py
testing/testsuite-targets.mk
toolkit/mozapps/installer/package-name.mk
toolkit/mozapps/installer/upload-files.mk
--- a/b2g/config/emulator-ics/config.json
+++ b/b2g/config/emulator-ics/config.json
@@ -5,16 +5,17 @@
     "mock_packages": ["ccache", "make", "bison", "flex", "gcc", "g++", "mpfr", "zlib-devel", "ncurses-devel", "zip", "autoconf213", "glibc-static", "perl-Digest-SHA", "wget", "alsa-lib", "atk", "cairo", "dbus-glib", "fontconfig", "freetype", "glib2", "gtk2", "libXRender", "libXt", "pango", "mozilla-python27-mercurial", "openssh-clients", "nss-devel", "glibc-devel.i686", "libstdc++.i686", "zlib-devel.i686", "ncurses-devel.i686", "libX11-devel.i686", "mesa-libGL-devel.i686", "mesa-libGL-devel", "libX11-devel", "git", "libxml2"],
     "mock_files": [["/home/cltbld/.ssh", "/home/mock_mozilla/.ssh"]],
     "build_targets": ["droid", "package-emulator", "package-tests"],
     "upload_files": [
         "{workdir}/out/target/product/generic/*.tar.bz2",
         "{workdir}/out/target/product/generic/tests/*.zip",
         "{workdir}/out/emulator.tar.gz",
         "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
+        "{objdir}/dist/test_packages.json",
         "{workdir}/sources.xml"
     ],
     "upload_platform": "emulator-ics",
     "gecko_l10n_root": "https://hg.mozilla.org/l10n-central",
     "gaia": {
         "l10n": {
             "vcs": "hgtool",
             "root": "https://hg.mozilla.org/gaia-l10n"
--- a/b2g/config/emulator/config.json
+++ b/b2g/config/emulator/config.json
@@ -5,16 +5,17 @@
     "mock_packages": ["ccache", "make", "bison", "flex", "gcc", "g++", "mpfr", "zlib-devel", "ncurses-devel", "zip", "autoconf213", "glibc-static", "perl-Digest-SHA", "wget", "alsa-lib", "atk", "cairo", "dbus-glib", "fontconfig", "freetype", "glib2", "gtk2", "libXRender", "libXt", "pango", "mozilla-python27-mercurial", "openssh-clients", "nss-devel", "glibc-devel.i686", "libstdc++.i686", "zlib-devel.i686", "ncurses-devel.i686", "libX11-devel.i686", "mesa-libGL-devel.i686", "mesa-libGL-devel", "libX11-devel", "git", "libxml2"],
     "mock_files": [["/home/cltbld/.ssh", "/home/mock_mozilla/.ssh"]],
     "build_targets": ["droid", "package-emulator", "package-tests"],
     "upload_files": [
         "{workdir}/out/target/product/generic/*.tar.bz2",
         "{workdir}/out/target/product/generic/tests/*.zip",
         "{workdir}/out/emulator.tar.gz",
         "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
+        "{objdir}/dist/test_packages.json",
         "{workdir}/sources.xml"
     ],
     "public_upload_files": [
         "{workdir}/out/target/product/generic/*.tar.bz2",
         "{workdir}/out/target/product/generic/tests/*.zip",
         "{objdir}/dist/b2g-*.crashreporter-symbols.zip",
         "{objdir}/dist/b2g-*.tar.gz",
         "{workdir}/sources.xml"
--- a/build/gen_mach_buildprops.py
+++ b/build/gen_mach_buildprops.py
@@ -51,20 +51,22 @@ def getUrlProperties(filename):
         ('unsignedApkUrl', lambda m: m.endswith('apk') and
                            'unsigned-unaligned' in m),
         ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m),
         ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
         ('completeMarUrl', lambda m: m.endswith('.complete.mar')),
         ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m),
         ('codeCoverageURL', lambda m: m.endswith('code-coverage-gcno.zip')),
         ('sdkUrl', lambda m: m.endswith(('sdk.tar.bz2', 'sdk.zip'))),
+        ('testPackagesUrl', lambda m: m.endswith('test_packages.json')),
         # packageUrl must be last!
-        ('packageUrl', lambda m: True),
+        ('packageUrl', lambda m: (not m.endswith('.json') and
+                                  not m.endswith('tests.zip'))),
     ]
-    url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''')
+    url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz|json))$''')
     properties = {}
 
     try:
         with open(filename) as f:
             for line in f:
                 m = url_re.match(line)
                 if m:
                     m = m.group(1)
new file mode 100644
--- /dev/null
+++ b/build/gen_test_packages_manifest.py
@@ -0,0 +1,63 @@
+#!/usr/bin/python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+
+from argparse import ArgumentParser
+
+ALL_HARNESSES = [
+    'common', # Harnesses without a specific package will look here.
+    'mochitest',
+    'reftest',
+    'webapprt',
+    'xpcshell',
+    'cppunittest',
+    'jittest',
+    'mozbase',
+]
+
+PACKAGE_SPECIFIED_HARNESSES = [
+]
+
+
+def parse_args():
+    parser = ArgumentParser(description='Generate a test_packages.json file to tell automation which harnesses require which test packages.')
+    parser.add_argument("--common", required=True,
+                        action="store", dest="tests_common",
+                        help="Name of the \"common\" archive, a package to be used by all harnesses.")
+    parser.add_argument("--jsshell", required=True,
+                        action="store", dest="jsshell",
+                        help="Name of the jsshell zip.")
+    for harness in PACKAGE_SPECIFIED_HARNESSES:
+        parser.add_argument("--%s" % harness, required=True,
+                            action="store", dest=harness,
+                            help="Name of the %s zip." % harness)
+    parser.add_argument("--dest-file", required=True,
+                        action="store", dest="destfile",
+                        help="Path to the output file to be written.")
+    return parser.parse_args()
+
+def generate_package_data(args):
+    # Generate a dictionary mapping test harness names (exactly as they're known to
+    # mozharness and testsuite-targets.mk, ideally) to the set of archive names that
+    # harness depends on to run.
+    # mozharness will use this file to determine what test zips to download,
+    # which will be an optimization once parts of the main zip are split to harness
+    # specific zips.
+    tests_common = args.tests_common
+    jsshell = args.jsshell
+
+    harness_requirements = dict([(k, [tests_common]) for k in ALL_HARNESSES])
+    harness_requirements['jittest'].append(jsshell)
+    for harness in PACKAGE_SPECIFIED_HARNESSES:
+        harness_requirements[harness].append(getattr(args, harness))
+    return harness_requirements
+
+if __name__ == '__main__':
+    args = parse_args()
+    packages_data = generate_package_data(args)
+    with open(args.destfile, 'w') as of:
+        json.dump(packages_data, of, indent=4)
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -401,25 +401,30 @@ package-tests: \
   stage-mozbase \
   stage-tps \
   stage-modules \
   stage-marionette \
   stage-cppunittests \
   stage-jittest \
   stage-web-platform-tests \
   stage-luciddream \
+  test-packages-manifest \
   $(NULL)
 ifdef MOZ_WEBRTC
 package-tests: stage-steeplechase
 endif
 else
 # This staging area has been built for us by universal/flight.mk
 PKG_STAGE = $(DIST)/universal/test-stage
 endif
 
+test-packages-manifest:
+	@rm -f $(MOZ_TEST_PACKAGES_FILE)
+	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py --common $(TEST_PACKAGE) --jsshell $(JSSHELL_NAME) --dest-file $(MOZ_TEST_PACKAGES_FILE)
+
 package-tests:
 	@rm -f '$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)'
 ifndef UNIVERSAL_BINARY
 	$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
 endif
 	$(MKDIR) -p $(abspath $(DIST))/$(PKG_PATH) && \
 	cd $(PKG_STAGE) && \
 	  zip -rq9D '$(abspath $(DIST))/$(PKG_PATH)$(TEST_PACKAGE)' \
@@ -589,10 +594,11 @@ stage-instrumentation-tests: make-stage-
   stage-mozbase \
   stage-tps \
   stage-modules \
   stage-marionette \
   stage-steeplechase \
   stage-web-platform-tests \
   stage-instrumentation-tests \
   stage-luciddream \
+  test-packages-manifest \
   $(NULL)
 
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -150,13 +150,15 @@ MOZ_SOURCE_STAMP = $(firstword $(shell h
 ###########################################################################
 ifdef MOZ_INCLUDE_SOURCE_INFO
 MOZ_SOURCE_REPO = $(call getSourceRepo,$(MOZILLA_DIR)$(NULL) $(NULL))
 endif
 
 MOZ_SOURCESTAMP_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).txt
 MOZ_BUILDINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).json
 MOZ_MOZINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).mozinfo.json
+MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/test_packages.json
 
 # JavaScript Shell
-PKG_JSSHELL = $(DIST)/jsshell-$(MOZ_PKG_PLATFORM).zip
+JSSHELL_NAME = jsshell-$(MOZ_PKG_PLATFORM).zip
+PKG_JSSHELL = $(DIST)/$(JSSHELL_NAME)
 
 endif # PACKAGE_NAME_MK_INCLUDED
--- a/toolkit/mozapps/installer/upload-files.mk
+++ b/toolkit/mozapps/installer/upload-files.mk
@@ -721,16 +721,17 @@ UPLOAD_FILES= \
   $(call QUOTED_WILDCARD,$(DIST)/$(LANGPACK)) \
   $(call QUOTED_WILDCARD,$(wildcard $(DIST)/$(PARTIAL_MAR))) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)) \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip) \
   $(call QUOTED_WILDCARD,$(DIST)/$(SDK)) \
   $(call QUOTED_WILDCARD,$(MOZ_SOURCESTAMP_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_BUILDINFO_FILE)) \
   $(call QUOTED_WILDCARD,$(MOZ_MOZINFO_FILE)) \
+  $(call QUOTED_WILDCARD,$(MOZ_TEST_PACKAGES_FILE)) \
   $(call QUOTED_WILDCARD,$(PKG_JSSHELL)) \
   $(if $(UPLOAD_EXTRA_FILES), $(foreach f, $(UPLOAD_EXTRA_FILES), $(wildcard $(DIST)/$(f))))
 
 ifdef MOZ_CRASHREPORTER_UPLOAD_FULL_SYMBOLS
 UPLOAD_FILES += \
   $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip)
 endif