Bug 1397043 - Update mozharness so that no test package is downloaded more than once. r=jmaher, a=test-only
authorGeoff Brown <gbrown@mozilla.com>
Thu, 07 Sep 2017 13:17:43 -0600
changeset 424000 f81f90c4fcb3bef81ca3a06dc79d08ed8dce21d3
parent 423999 2dee3996eee2576ca14615b0f6e24ce496a2fb39
child 424001 f01d58e60d69ab8352a6bfee9cb3a7f03176e2a4
push id1517
push userjlorenzo@mozilla.com
push dateThu, 14 Sep 2017 16:50:54 +0000
treeherdermozilla-release@3b41fd564418 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher, test-only
bugs1397043
milestone56.0
Bug 1397043 - Update mozharness so that no test package is downloaded more than once. r=jmaher, a=test-only
testing/mozharness/mozharness/mozilla/testing/testbase.py
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -446,49 +446,52 @@ 2. running via buildbot and running the 
         }
         suite_categories = [aliases.get(name, name) for name in suite_categories]
 
         dirs = self.query_abs_dirs()
         test_install_dir = dirs.get('abs_test_install_dir',
                                     os.path.join(dirs['abs_work_dir'], 'tests'))
         self.mkdir_p(test_install_dir)
         package_requirements = self._read_packages_manifest()
+        target_packages = []
         for category in suite_categories:
             if category in package_requirements:
-                target_packages = package_requirements[category]
+                target_packages.extend(package_requirements[category])
             else:
                 # If we don't harness specific requirements, assume the common zip
                 # has everything we need to run tests for this suite.
-                target_packages = package_requirements['common']
+                target_packages.extend(package_requirements['common'])
 
-            self.info("Downloading packages: %s for test suite category: %s" %
-                      (target_packages, category))
-            for file_name in target_packages:
-                target_dir = test_install_dir
-                unpack_dirs = extract_dirs
+        # eliminate duplicates -- no need to download anything twice
+        target_packages = list(set(target_packages))
+        self.info("Downloading packages: %s for test suite categories: %s" %
+                  (target_packages, suite_categories))
+        for file_name in target_packages:
+            target_dir = test_install_dir
+            unpack_dirs = extract_dirs
 
-                if "common.tests" in file_name and isinstance(unpack_dirs, list):
-                    # Ensure that the following files are always getting extracted
-                    required_files = ["mach",
-                                      "mozinfo.json",
-                                      ]
-                    for req_file in required_files:
-                        if req_file not in unpack_dirs:
-                            self.info("Adding '{}' for extraction from common.tests zip file"
-                                      .format(req_file))
-                            unpack_dirs.append(req_file)
+            if "common.tests" in file_name and isinstance(unpack_dirs, list):
+                # Ensure that the following files are always getting extracted
+                required_files = ["mach",
+                                  "mozinfo.json",
+                                  ]
+                for req_file in required_files:
+                    if req_file not in unpack_dirs:
+                        self.info("Adding '{}' for extraction from common.tests zip file"
+                                  .format(req_file))
+                        unpack_dirs.append(req_file)
 
-                if "jsshell-" in file_name or file_name == "target.jsshell.zip":
-                    self.info("Special-casing the jsshell zip file")
-                    unpack_dirs = None
-                    target_dir = dirs['abs_test_bin_dir']
+            if "jsshell-" in file_name or file_name == "target.jsshell.zip":
+                self.info("Special-casing the jsshell zip file")
+                unpack_dirs = None
+                target_dir = dirs['abs_test_bin_dir']
 
-                url = self.query_build_dir_url(file_name)
-                self.download_unpack(url, target_dir,
-                                     extract_dirs=unpack_dirs)
+            url = self.query_build_dir_url(file_name)
+            self.download_unpack(url, target_dir,
+                                 extract_dirs=unpack_dirs)
 
     def _download_test_zip(self, extract_dirs=None):
         dirs = self.query_abs_dirs()
         test_install_dir = dirs.get('abs_test_install_dir',
                                     os.path.join(dirs['abs_work_dir'], 'tests'))
         self.download_unpack(self.test_url, test_install_dir,
                              extract_dirs=extract_dirs)