Backed out changeset 889072501f26 (bug 1239808) bustage for web platform tests as run via taskcluster
authorCarsten "Tomcat" Book <cbook@mozilla.com>
Wed, 24 Feb 2016 12:52:16 +0100
changeset 321716 a097eb2ace2276e8f34477cacb061a7360fdf4e5
parent 321715 4894d75bf376b60766430de3b76106023bab4ac3
child 321717 94abde4bfd895edba6436f90d328da5e1c0e8610
push id5913
push userjlund@mozilla.com
push dateMon, 25 Apr 2016 16:57:49 +0000
treeherdermozilla-beta@dcaf0a6fa115 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1239808
milestone47.0a1
backs out889072501f26d2e89e3d3552e7b4fad79cbffb75
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset 889072501f26 (bug 1239808) bustage for web platform tests as run via taskcluster
build/gen_test_packages_manifest.py
testing/mozharness/mozharness/mozilla/testing/testbase.py
testing/testsuite-targets.mk
toolkit/mozapps/installer/package-name.mk
--- a/build/gen_test_packages_manifest.py
+++ b/build/gen_test_packages_manifest.py
@@ -40,46 +40,53 @@ OPTIONAL_PACKAGES = [
 def parse_args():
     parser = ArgumentParser(description='Generate a test_packages.json file to tell automation which harnesses require which test packages.')
     parser.add_argument("--common", required=True,
                         action="store", dest="tests_common",
                         help="Name of the \"common\" archive, a package to be used by all harnesses.")
     parser.add_argument("--jsshell", required=True,
                         action="store", dest="jsshell",
                         help="Name of the jsshell zip.")
+    parser.add_argument("--use-short-names", action="store_true",
+                        help="Use short names for packages (target.$name.tests.zip "
+                             "instead of $(PACKAGE_BASENAME).$name.tests.zip)")
     for harness in PACKAGE_SPECIFIED_HARNESSES:
         parser.add_argument("--%s" % harness, required=True,
                             action="store", dest=harness,
                             help="Name of the %s zip." % harness)
     for harness in OPTIONAL_PACKAGES:
         parser.add_argument("--%s" % harness, required=False,
                             action="store", dest=harness,
                             help="Name of the %s zip." % harness)
     parser.add_argument("--dest-file", required=True,
                         action="store", dest="destfile",
                         help="Path to the output file to be written.")
     return parser.parse_args()
 
-
 def generate_package_data(args):
     # Generate a dictionary mapping test harness names (exactly as they're known to
     # mozharness and testsuite-targets.mk, ideally) to the set of archive names that
     # harness depends on to run.
     # mozharness will use this file to determine what test zips to download,
     # which will be an optimization once parts of the main zip are split to harness
     # specific zips.
     tests_common = args.tests_common
+    if args.use_short_names:
+        tests_common = 'target.common.tests.zip'
+
     jsshell = args.jsshell
 
     harness_requirements = dict([(k, [tests_common]) for k in ALL_HARNESSES])
     harness_requirements['jittest'].append(jsshell)
     for harness in PACKAGE_SPECIFIED_HARNESSES + OPTIONAL_PACKAGES:
         pkg_name = getattr(args, harness, None)
         if pkg_name is None:
             continue
+        if args.use_short_names:
+            pkg_name = 'target.%s.tests.zip' % harness
         harness_requirements[harness].append(pkg_name)
     return harness_requirements
 
 if __name__ == '__main__':
     args = parse_args()
     packages_data = generate_package_data(args)
     with open(args.destfile, 'w') as of:
         json.dump(packages_data, of, indent=4)
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -159,44 +159,26 @@ class TestingMixin(VirtualenvMixin, Buil
             self.fatal("Can't figure out build directory urls without an installer_url "
                        "or test_packages_url!")
 
         last_slash = reference_url.rfind('/')
         base_url = reference_url[:last_slash]
 
         return '%s/%s' % (base_url, file_name)
 
-    def query_prefixed_build_dir_url(self, suffix):
-        """Resolve a file name prefixed with platform and build details to a potential url
-        in the build upload directory where that file can be found.
-        """
-        if self.test_packages_url:
-            reference_suffixes = ['.test_packages.json']
-            reference_url = self.test_packages_url
-        elif self.installer_url:
-            reference_suffixes = INSTALLER_SUFFIXES
-            reference_url = self.installer_url
-        else:
-            self.fatal("Can't figure out build directory urls without an installer_url "
-                       "or test_packages_url!")
-
-        url = None
-        for reference_suffix in reference_suffixes:
-            if reference_url.endswith(reference_suffix):
-                url = reference_url[:-len(reference_suffix)] + suffix
-                break
-
-        return url
-
     def query_symbols_url(self):
         if self.symbols_url:
             return self.symbols_url
 
         elif self.installer_url:
-            symbols_url = self.query_prefixed_build_dir_url('.crashreporter-symbols.zip')
+            symbols_url = None
+            for suffix in INSTALLER_SUFFIXES:
+                if self.installer_url.endswith(suffix):
+                    symbols_url = self.installer_url[:-len(suffix)] + '.crashreporter-symbols.zip'
+                    break
 
             # Check if the URL exists. If not, use none to allow mozcrash to auto-check for symbols
             try:
                 if symbols_url:
                     self._urlopen(symbols_url)
                     self.symbols_url = symbols_url
             except urllib2.URLError:
                 self.warning("Can't figure out symbols_url from installer_url: %s!" %
@@ -549,17 +531,17 @@ 2. running via buildbot and running the 
 
             self._download_test_zip(target_unzip_dirs)
         else:
             if not self.test_packages_url:
                 # The caller intends to download harness specific packages, but doesn't know
                 # where the packages manifest is located. This is the case when the
                 # test package manifest isn't set as a buildbot property, which is true
                 # for some self-serve jobs and platforms using parse_make_upload.
-                self.test_packages_url = self.query_prefixed_build_dir_url('.test_packages.json')
+                self.test_packages_url = self.query_build_dir_url('test_packages.json')
 
             suite_categories = suite_categories or ['common']
             self._download_test_packages(suite_categories, target_unzip_dirs)
 
         self._download_installer()
         if self.config.get('download_symbols'):
             self._download_and_extract_symbols()
 
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -261,16 +261,17 @@ stage-all: \
   stage-mochitest \
   stage-xpcshell \
   stage-jstests \
   stage-jetpack \
   stage-marionette \
   stage-cppunittests \
   stage-luciddream \
   test-packages-manifest \
+  test-packages-manifest-tc \
   $(NULL)
 ifdef MOZ_WEBRTC
 stage-all: stage-steeplechase
 endif
 
 TEST_PKGS := \
   common \
   cppunittest \
@@ -283,16 +284,26 @@ TEST_PKGS := \
 
 ifdef BUILD_GTEST
 stage-all: stage-gtest
 TEST_PKGS += gtest
 endif
 
 PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.zip'
 
+test-packages-manifest-tc:
+	@rm -f $(MOZ_TEST_PACKAGES_FILE_TC)
+	$(NSINSTALL) -D $(dir $(MOZ_TEST_PACKAGES_FILE_TC))
+	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
+      --jsshell $(JSSHELL_NAME) \
+      --dest-file $(MOZ_TEST_PACKAGES_FILE_TC) \
+      --use-short-names \
+      $(call PKG_ARG,common) \
+      $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
+
 test-packages-manifest:
 	@rm -f $(MOZ_TEST_PACKAGES_FILE)
 	$(NSINSTALL) -D $(dir $(MOZ_TEST_PACKAGES_FILE))
 	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
       --jsshell $(JSSHELL_NAME) \
       --dest-file $(MOZ_TEST_PACKAGES_FILE) \
       $(call PKG_ARG,common) \
       $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
@@ -462,9 +473,11 @@ stage-extensions: make-stage-dir
   stage-jstests \
   stage-android \
   stage-jetpack \
   stage-marionette \
   stage-steeplechase \
   stage-instrumentation-tests \
   stage-luciddream \
   test-packages-manifest \
+  test-packages-manifest-tc \
   $(NULL)
+
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -151,17 +151,18 @@ ifndef INCLUDED_RCS_MK
   USE_RCS_MK := 1
   include $(MOZILLA_DIR)/config/makefiles/makeutils.mk
 endif
 
 MOZ_SOURCESTAMP_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).txt
 MOZ_BUILDINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).json
 MOZ_BUILDID_INFO_TXT_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME)_info.txt
 MOZ_MOZINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).mozinfo.json
-MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/$(PKG_BASENAME).test_packages.json
+MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/test_packages.json
+MOZ_TEST_PACKAGES_FILE_TC = $(DIST)/$(PKG_PATH)/test_packages_tc.json
 
 # JavaScript Shell
 ifdef MOZ_SIMPLE_PACKAGE_NAME
 JSSHELL_NAME := $(MOZ_SIMPLE_PACKAGE_NAME).jsshell.zip
 else
 JSSHELL_NAME = jsshell-$(MOZ_PKG_PLATFORM).zip
 endif
 PKG_JSSHELL = $(DIST)/$(JSSHELL_NAME)