Bug 1640758 - Use multiple errorsummary and raw logs for test-verify; r=jmaher
authorGeoff Brown <gbrown@mozilla.com>
Tue, 02 Jun 2020 22:56:14 +0000
changeset 533636 db1e66399e7e55b0253232fad520305d6b06a0b8
parent 533635 74e66b1ca7a31665ff19d0322f5ff76d2b20f3d8
child 533637 ca779f6cf743c1525bb35b6a8be465a87d700527
push id37475
push userbtara@mozilla.com
push dateWed, 03 Jun 2020 16:12:12 +0000
treeherdermozilla-central@ea34fd156c89 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1640758
milestone79.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1640758 - Use multiple errorsummary and raw logs for test-verify; r=jmaher In all mozharness scripts running test-verify (or per-test coverage), use distinct file names for errorsummary and raw logs. This usually means that the command needs to be built inside the per-test loop. Differential Revision: https://phabricator.services.mozilla.com/D77746
testing/mozharness/mozharness/mozilla/testing/per_test_base.py
testing/mozharness/scripts/android_emulator_unittest.py
testing/mozharness/scripts/android_hardware_unittest.py
testing/mozharness/scripts/desktop_unittest.py
testing/mozharness/scripts/web_platform_tests.py
--- a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
+++ b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
@@ -23,16 +23,17 @@ class SingleTestMixin(object):
 
         self.suites = {}
         self.tests_downloaded = False
         self.reftest_test_dir = None
         self.jsreftest_test_dir = None
         # Map from full test path on the test machine to a relative path in the source checkout.
         # Use self._map_test_path_to_source(test_machine_path, source_path) to add a mapping.
         self.test_src_path = {}
+        self.per_test_log_index = 1
 
     def _map_test_path_to_source(self, test_machine_path, source_path):
         test_machine_path = test_machine_path.replace(os.sep, posixpath.sep)
         source_path = source_path.replace(os.sep, posixpath.sep)
         self.test_src_path[test_machine_path] = source_path
 
     def _is_gpu_suite(self, suite):
         if suite and (suite == 'gpu' or suite.startswith('webgl')):
@@ -391,8 +392,21 @@ class SingleTestMixin(object):
             while len(new) < max_test_name_len:
                 head, tail = os.path.split(head)
                 previous = new
                 new = os.path.join(tail, new)
             test_name = os.path.join('...', previous or new)
             test_name = test_name.rstrip(os.path.sep)
         self.log("TinderboxPrint: Per-test run of %s<br/>: %s" %
                  (test_name, tbpl_status), level=log_level)
+
+    def get_indexed_logs(self, dir, test_suite):
+        """
+           Per-test tasks need distinct file names for the raw and errorsummary logs
+           on each run.
+        """
+        index = ''
+        if self.verify_enabled or self.per_test_coverage:
+            index = '-test%d' % self.per_test_log_index
+            self.per_test_log_index += 1
+        raw_log_file = os.path.join(dir, '%s%s_raw.log' % (test_suite, index))
+        error_summary_file = os.path.join(dir, '%s%s_errorsummary.log' % (test_suite, index))
+        return raw_log_file, error_summary_file
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -218,21 +218,19 @@ class AndroidEmulatorTest(TestingMixin, 
             self.query_python_path('python'),
             '-u',
             os.path.join(
                 self._query_tests_dir(self.test_suite),
                 self.config["suite_definitions"][self.test_suite]["run_filename"]
             ),
         ]
 
-        raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                    '%s_raw.log' % self.test_suite)
+        raw_log_file, error_summary_file = self.get_indexed_logs(dirs['abs_blob_upload_dir'],
+                                                                 self.test_suite)
 
-        error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                          '%s_errorsummary.log' % self.test_suite)
         str_format_values = {
             'device_serial': self.device_serial,
             # IP address of the host as seen from the emulator
             'remote_webserver': '10.0.2.2',
             'xre_path': self.xre_path,
             'utility_path': self.xre_path,
             'http_port': '8854',  # starting http port  to use for the mochitest server
             'ssl_port': '4454',  # starting ssl port to use for the server
@@ -399,18 +397,16 @@ class AndroidEmulatorTest(TestingMixin, 
         max_per_test_time = datetime.timedelta(minutes=60)
 
         per_test_args = []
         suites = self._query_suites()
         minidump = self.query_minidump_stackwalk()
         for (per_test_suite, suite) in suites:
             self.test_suite = suite
 
-            cmd = self._build_command()
-
             try:
                 cwd = self._query_tests_dir(self.test_suite)
             except Exception:
                 self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite)
 
             env = self.query_env()
             if minidump:
                 env['MINIDUMP_STACKWALK'] = minidump
@@ -427,16 +423,17 @@ class AndroidEmulatorTest(TestingMixin, 
                     # them so that a task timeout is not triggered, and so that
                     # (partial) results are made available in a timely manner.
                     self.info("TinderboxPrint: Running tests took too long: "
                               "Not all tests were executed.<br/>")
                     # Signal per-test time exceeded, to break out of suites and
                     # suite categories loops also.
                     return
 
+                cmd = self._build_command()
                 final_cmd = copy.copy(cmd)
                 if len(per_test_args) > 0:
                     # in per-test mode, remove any chunk arguments from command
                     for arg in final_cmd:
                         if 'total-chunk' in arg or 'this-chunk' in arg:
                             final_cmd.remove(arg)
                 final_cmd.extend(per_test_args)
 
--- a/testing/mozharness/scripts/android_hardware_unittest.py
+++ b/testing/mozharness/scripts/android_hardware_unittest.py
@@ -188,21 +188,19 @@ class AndroidHardwareTest(TestingMixin, 
             self.query_python_path('python'),
             '-u',
             os.path.join(
                 self._query_tests_dir(),
                 self.config["suite_definitions"][self.test_suite]["run_filename"]
             ),
         ]
 
-        raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                    '%s_raw.log' % self.test_suite)
+        raw_log_file, error_summary_file = self.get_indexed_logs(dirs['abs_blob_upload_dir'],
+                                                                 self.test_suite)
 
-        error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                          '%s_errorsummary.log' % self.test_suite)
         str_format_values = {
             'device_serial': self.device_serial,
             'remote_webserver': c['remote_webserver'],
             'xre_path': self.xre_path,
             'utility_path': self.xre_path,
             'http_port': '8854',  # starting http port  to use for the mochitest server
             'ssl_port': '4454',  # starting ssl port to use for the server
             'certs_path': os.path.join(dirs['abs_work_dir'], 'tests/certs'),
@@ -358,18 +356,16 @@ class AndroidHardwareTest(TestingMixin, 
         max_per_test_time = datetime.timedelta(minutes=60)
 
         per_test_args = []
         suites = self._query_suites()
         minidump = self.query_minidump_stackwalk()
         for (per_test_suite, suite) in suites:
             self.test_suite = suite
 
-            cmd = self._build_command()
-
             try:
                 cwd = self._query_tests_dir()
             except Exception:
                 self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite)
             env = self.query_env()
             if minidump:
                 env['MINIDUMP_STACKWALK'] = minidump
             env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
@@ -383,16 +379,17 @@ class AndroidHardwareTest(TestingMixin, 
                     # them so that a task timeout is not triggered, and so that
                     # (partial) results are made available in a timely manner.
                     self.info("TinderboxPrint: Running tests took too long: "
                               "Not all tests were executed.<br/>")
                     # Signal per-test time exceeded, to break out of suites and
                     # suite categories loops also.
                     return
 
+                cmd = self._build_command()
                 final_cmd = copy.copy(cmd)
                 if len(per_test_args) > 0:
                     # in per-test mode, remove any chunk arguments from command
                     for arg in final_cmd:
                         if 'total-chunk' in arg or 'this-chunk' in arg:
                             final_cmd.remove(arg)
                 final_cmd.extend(per_test_args)
 
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -376,21 +376,19 @@ class DesktopUnittest(TestingMixin, Merc
             c = self.config
             dirs = self.query_abs_dirs()
             run_file = c['run_file_names'][suite_category]
             base_cmd = [self.query_python_path('python'), '-u']
             base_cmd.append(os.path.join(dirs["abs_%s_dir" % suite_category], run_file))
             abs_app_dir = self.query_abs_app_dir()
             abs_res_dir = self.query_abs_res_dir()
 
-            raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                        '%s_raw.log' % suite)
+            raw_log_file, error_summary_file = self.get_indexed_logs(
+                dirs['abs_blob_upload_dir'], suite)
 
-            error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
-                                              '%s_errorsummary.log' % suite)
             str_format_values = {
                 'binary_path': self.binary_path,
                 'symbols_path': self._query_symbols_url(),
                 'abs_work_dir': dirs['abs_work_dir'],
                 'abs_app_dir': abs_app_dir,
                 'abs_res_dir': abs_res_dir,
                 'raw_log_file': raw_log_file,
                 'error_summary_file': error_summary_file,
@@ -812,18 +810,16 @@ class DesktopUnittest(TestingMixin, Merc
         executed_too_many_tests = False
 
         if suites:
             self.info('#### Running %s suites' % suite_category)
             for suite in suites:
                 if executed_too_many_tests and not self.per_test_coverage:
                     return False
 
-                abs_base_cmd = self._query_abs_base_cmd(suite_category, suite)
-                cmd = abs_base_cmd[:]
                 replace_dict = {
                     'abs_app_dir': abs_app_dir,
 
                     # Mac specific, but points to abs_app_dir on other
                     # platforms.
                     'abs_res_dir': abs_res_dir,
                 }
                 options_list = []
@@ -841,23 +837,16 @@ class DesktopUnittest(TestingMixin, Merc
                     env = copy.deepcopy(suites[suite].get('env', {}))
                 else:
                     options_list = suites[suite]
                     tests_list = []
 
                 flavor = self._query_try_flavor(suite_category, suite)
                 try_options, try_tests = self.try_args(flavor)
 
-                cmd.extend(self.query_options(options_list,
-                                              try_options,
-                                              str_format_values=replace_dict))
-                cmd.extend(self.query_tests_args(tests_list,
-                                                 try_tests,
-                                                 str_format_values=replace_dict))
-
                 suite_name = suite_category + '-' + suite
                 tbpl_status, log_level = None, None
                 error_list = BaseErrorList + HarnessErrorList
                 parser = self.get_test_output_parser(suite_category,
                                                      flavor=flavor,
                                                      config=self.config,
                                                      error_list=error_list,
                                                      log_obj=self.log_obj)
@@ -913,16 +902,25 @@ class DesktopUnittest(TestingMixin, Merc
                             # to run all tests, and attempting to do so may cause other
                             # problems, such as generating too much log output.
                             self.info("TinderboxPrint: Too many modified tests: Not all tests "
                                       "were executed.<br/>")
                             executed_too_many_tests = True
 
                         executed_tests = executed_tests + 1
 
+                    abs_base_cmd = self._query_abs_base_cmd(suite_category, suite)
+                    cmd = abs_base_cmd[:]
+                    cmd.extend(self.query_options(options_list,
+                                                  try_options,
+                                                  str_format_values=replace_dict))
+                    cmd.extend(self.query_tests_args(tests_list,
+                                                     try_tests,
+                                                     str_format_values=replace_dict))
+
                     final_cmd = copy.copy(cmd)
                     final_cmd.extend(per_test_args)
 
                     final_env = copy.copy(env)
 
                     if self.per_test_coverage:
                         self.set_coverage_env(final_env)
 
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -222,23 +222,24 @@ class WebPlatformTest(TestingMixin, Merc
             'xre_path': self.xre_path,
         }
 
         cmd = [self.query_python_path('python'), '-u']
         cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))
 
         mozinfo.find_and_update_from_json(dirs['abs_test_install_dir'])
 
+        raw_log_file, error_summary_file = self.get_indexed_logs(dirs['abs_blob_upload_dir'],
+                                                                 'wpt')
+
         cmd += ["--log-raw=-",
-                "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"],
-                                              "wpt_raw.log"),
+                "--log-raw=%s" % raw_log_file,
                 "--log-wptreport=%s" % os.path.join(dirs["abs_blob_upload_dir"],
                                                     "wptreport.json"),
-                "--log-errorsummary=%s" % os.path.join(dirs["abs_blob_upload_dir"],
-                                                       "wpt_errorsummary.log"),
+                "--log-errorsummary=%s" % error_summary_file,
                 "--binary=%s" % self.binary_path,
                 "--symbols-path=%s" % self.symbols_path,
                 "--stackwalk-binary=%s" % self.query_minidump_stackwalk(),
                 "--stackfix-dir=%s" % os.path.join(dirs["abs_test_install_dir"], "bin"),
                 "--run-by-dir=%i" % (3 if not mozinfo.info["asan"] else 0),
                 "--no-pause-after-test",
                 "--instrument-to-file=%s" % os.path.join(dirs["abs_blob_upload_dir"],
                                                          "wpt_instruments.txt")]