Bug 1239808 - Rename test_packages.json to include package basename as prefix. r=chmanchester,jlund
authorHenrik Skupin <mail@hskupin.info>
Thu, 25 Feb 2016 21:37:47 +0100
changeset 286073 dc76f89aa251948ad07d719a37d1b8a24c960eaf
parent 286072 e43bf2d10525625be8a23c8a87b8c5771bac6775
child 286074 f58db16c41c0907462b9b1c97dfb874f2a2511e8
push id30039
push usercbook@mozilla.com
push dateTue, 01 Mar 2016 11:02:11 +0000
treeherdermozilla-central@5cafa6f3019b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester, jlund
bugs1239808
milestone47.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1239808 - Rename test_packages.json to include package basename as prefix. r=chmanchester,jlund MozReview-Commit-ID: 6JiHIbSjhoR
build/gen_test_packages_manifest.py
testing/mozharness/mozharness/mozilla/testing/testbase.py
testing/testsuite-targets.mk
toolkit/mozapps/installer/package-name.mk
--- a/build/gen_test_packages_manifest.py
+++ b/build/gen_test_packages_manifest.py
@@ -40,53 +40,46 @@ OPTIONAL_PACKAGES = [
 def parse_args():
     parser = ArgumentParser(description='Generate a test_packages.json file to tell automation which harnesses require which test packages.')
     parser.add_argument("--common", required=True,
                         action="store", dest="tests_common",
                         help="Name of the \"common\" archive, a package to be used by all harnesses.")
     parser.add_argument("--jsshell", required=True,
                         action="store", dest="jsshell",
                         help="Name of the jsshell zip.")
-    parser.add_argument("--use-short-names", action="store_true",
-                        help="Use short names for packages (target.$name.tests.zip "
-                             "instead of $(PACKAGE_BASENAME).$name.tests.zip)")
     for harness in PACKAGE_SPECIFIED_HARNESSES:
         parser.add_argument("--%s" % harness, required=True,
                             action="store", dest=harness,
                             help="Name of the %s zip." % harness)
     for harness in OPTIONAL_PACKAGES:
         parser.add_argument("--%s" % harness, required=False,
                             action="store", dest=harness,
                             help="Name of the %s zip." % harness)
     parser.add_argument("--dest-file", required=True,
                         action="store", dest="destfile",
                         help="Path to the output file to be written.")
     return parser.parse_args()
 
+
 def generate_package_data(args):
     # Generate a dictionary mapping test harness names (exactly as they're known to
     # mozharness and testsuite-targets.mk, ideally) to the set of archive names that
     # harness depends on to run.
     # mozharness will use this file to determine what test zips to download,
     # which will be an optimization once parts of the main zip are split to harness
     # specific zips.
     tests_common = args.tests_common
-    if args.use_short_names:
-        tests_common = 'target.common.tests.zip'
-
     jsshell = args.jsshell
 
     harness_requirements = dict([(k, [tests_common]) for k in ALL_HARNESSES])
     harness_requirements['jittest'].append(jsshell)
     for harness in PACKAGE_SPECIFIED_HARNESSES + OPTIONAL_PACKAGES:
         pkg_name = getattr(args, harness, None)
         if pkg_name is None:
             continue
-        if args.use_short_names:
-            pkg_name = 'target.%s.tests.zip' % harness
         harness_requirements[harness].append(pkg_name)
     return harness_requirements
 
 if __name__ == '__main__':
     args = parse_args()
     packages_data = generate_package_data(args)
     with open(args.destfile, 'w') as of:
         json.dump(packages_data, of, indent=4)
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -159,26 +159,44 @@ class TestingMixin(VirtualenvMixin, Buil
             self.fatal("Can't figure out build directory urls without an installer_url "
                        "or test_packages_url!")
 
         last_slash = reference_url.rfind('/')
         base_url = reference_url[:last_slash]
 
         return '%s/%s' % (base_url, file_name)
 
+    def query_prefixed_build_dir_url(self, suffix):
+        """Resolve a file name prefixed with platform and build details to a potential url
+        in the build upload directory where that file can be found.
+        """
+        if self.test_packages_url:
+            reference_suffixes = ['.test_packages.json']
+            reference_url = self.test_packages_url
+        elif self.installer_url:
+            reference_suffixes = INSTALLER_SUFFIXES
+            reference_url = self.installer_url
+        else:
+            self.fatal("Can't figure out build directory urls without an installer_url "
+                       "or test_packages_url!")
+
+        url = None
+        for reference_suffix in reference_suffixes:
+            if reference_url.endswith(reference_suffix):
+                url = reference_url[:-len(reference_suffix)] + suffix
+                break
+
+        return url
+
     def query_symbols_url(self):
         if self.symbols_url:
             return self.symbols_url
 
         elif self.installer_url:
-            symbols_url = None
-            for suffix in INSTALLER_SUFFIXES:
-                if self.installer_url.endswith(suffix):
-                    symbols_url = self.installer_url[:-len(suffix)] + '.crashreporter-symbols.zip'
-                    break
+            symbols_url = self.query_prefixed_build_dir_url('.crashreporter-symbols.zip')
 
             # Check if the URL exists. If not, use none to allow mozcrash to auto-check for symbols
             try:
                 if symbols_url:
                     self._urlopen(symbols_url)
                     self.symbols_url = symbols_url
             except urllib2.URLError:
                 self.warning("Can't figure out symbols_url from installer_url: %s!" %
@@ -531,17 +549,17 @@ 2. running via buildbot and running the 
 
             self._download_test_zip(target_unzip_dirs)
         else:
             if not self.test_packages_url:
                 # The caller intends to download harness specific packages, but doesn't know
                 # where the packages manifest is located. This is the case when the
                 # test package manifest isn't set as a buildbot property, which is true
                 # for some self-serve jobs and platforms using parse_make_upload.
-                self.test_packages_url = self.query_build_dir_url('test_packages.json')
+                self.test_packages_url = self.query_prefixed_build_dir_url('.test_packages.json')
 
             suite_categories = suite_categories or ['common']
             self._download_test_packages(suite_categories, target_unzip_dirs)
 
         self._download_installer()
         if self.config.get('download_symbols'):
             self._download_and_extract_symbols()
 
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -177,17 +177,16 @@ stage-all: \
   stage-extensions \
   stage-mochitest \
   stage-jstests \
   stage-jetpack \
   stage-marionette \
   stage-cppunittests \
   stage-luciddream \
   test-packages-manifest \
-  test-packages-manifest-tc \
   $(NULL)
 ifdef MOZ_WEBRTC
 stage-all: stage-steeplechase
 endif
 
 TEST_PKGS := \
   common \
   cppunittest \
@@ -200,26 +199,16 @@ TEST_PKGS := \
 
 ifdef BUILD_GTEST
 stage-all: stage-gtest
 TEST_PKGS += gtest
 endif
 
 PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.zip'
 
-test-packages-manifest-tc:
-	@rm -f $(MOZ_TEST_PACKAGES_FILE_TC)
-	$(NSINSTALL) -D $(dir $(MOZ_TEST_PACKAGES_FILE_TC))
-	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
-      --jsshell $(JSSHELL_NAME) \
-      --dest-file $(MOZ_TEST_PACKAGES_FILE_TC) \
-      --use-short-names \
-      $(call PKG_ARG,common) \
-      $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
-
 test-packages-manifest:
 	@rm -f $(MOZ_TEST_PACKAGES_FILE)
 	$(NSINSTALL) -D $(dir $(MOZ_TEST_PACKAGES_FILE))
 	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
       --jsshell $(JSSHELL_NAME) \
       --dest-file $(MOZ_TEST_PACKAGES_FILE) \
       $(call PKG_ARG,common) \
       $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
@@ -384,11 +373,9 @@ stage-extensions: make-stage-dir
   stage-jstests \
   stage-android \
   stage-jetpack \
   stage-marionette \
   stage-steeplechase \
   stage-instrumentation-tests \
   stage-luciddream \
   test-packages-manifest \
-  test-packages-manifest-tc \
   $(NULL)
-
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -151,18 +151,17 @@ ifndef INCLUDED_RCS_MK
   USE_RCS_MK := 1
   include $(MOZILLA_DIR)/config/makefiles/makeutils.mk
 endif
 
 MOZ_SOURCESTAMP_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).txt
 MOZ_BUILDINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).json
 MOZ_BUILDID_INFO_TXT_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME)_info.txt
 MOZ_MOZINFO_FILE = $(DIST)/$(PKG_PATH)/$(MOZ_INFO_BASENAME).mozinfo.json
-MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/test_packages.json
-MOZ_TEST_PACKAGES_FILE_TC = $(DIST)/$(PKG_PATH)/test_packages_tc.json
+MOZ_TEST_PACKAGES_FILE = $(DIST)/$(PKG_PATH)/$(PKG_BASENAME).test_packages.json
 
 # JavaScript Shell
 ifdef MOZ_SIMPLE_PACKAGE_NAME
 JSSHELL_NAME := $(MOZ_SIMPLE_PACKAGE_NAME).jsshell.zip
 else
 JSSHELL_NAME = jsshell-$(MOZ_PKG_PLATFORM).zip
 endif
 PKG_JSSHELL = $(DIST)/$(JSSHELL_NAME)