Bug 1458730 - Test-Verify summarization fails on windows. r=gbrown
authorJoel Maher <jmaher@mozilla.com>
Thu, 10 May 2018 06:00:43 -0400
changeset 417710 193e46b398bf45521768bf11fffddbb854320eeb
parent 417709 27d3c10daf7e13028054a40514c476733f413ea6
child 417711 d3bf0a95e4e4a92dea0c97715d9adc1944fbba6c
push id33977
push userncsoregi@mozilla.com
push dateThu, 10 May 2018 16:43:24 +0000
treeherdermozilla-central@17db33b6a124 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1458730
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1458730 - Test-Verify summarization fails on windows. r=gbrown
testing/mozharness/mozharness/mozilla/testing/unittest.py
testing/mozharness/scripts/web_platform_tests.py
--- a/testing/mozharness/mozharness/mozilla/testing/unittest.py
+++ b/testing/mozharness/mozharness/mozilla/testing/unittest.py
@@ -61,41 +61,19 @@ class TestSummaryOutputParserHelper(Outp
         if m:
             try:
                 setattr(self, m.group(1), int(m.group(2)))
             except ValueError:
                 # ignore bad values
                 pass
 
     def evaluate_parser(self, return_code, success_codes=None, previous_summary=None):
-        """
-          We can run evaluate_parser multiple times, it will duplicate failures
-          and status which can mean that future tests will fail if a previous test fails.
-          When we have a previous summary, we want to do 2 things:
-            1) Remove previous data from the new summary to only look at new data
-            2) Build a joined summary to include the previous + new data
-        """
-        keys = ['passed', 'failed']
-        joined_summary = {}
-        for key in keys:
-            joined_summary[key] = getattr(self, key)
-
-        if previous_summary:
-            for key in keys:
-                joined_summary[key] += previous_summary[key]
-                value = getattr(self, key) - previous_summary[key]
-                if value < 0:
-                    value = 0
-                setattr(self, key, value)
-            self.tbpl_status = TBPL_SUCCESS
-            self.worst_log_level = INFO
-
-        joined_summary = {}
-        if previous_summary:
-            joined_summary = previous_summary
+        # TestSummaryOutputParserHelper is for Marionette, which doesn't support test-verify
+        # When it does we can reset the internal state variables as needed
+        joined_summary = previous_summary
 
         if return_code == 0 and self.passed > 0 and self.failed == 0:
             self.tbpl_status = TBPL_SUCCESS
         elif return_code == 10 and self.failed > 0:
             self.tbpl_status = TBPL_WARNING
         else:
             self.tbpl_status = TBPL_FAILURE
             self.worst_log_level = ERROR
@@ -201,34 +179,25 @@ class DesktopUnittestOutputParser(Output
 
         if self.num_errors:  # mozharness ran into a script error
             self.tbpl_status = self.worst_level(TBPL_FAILURE, self.tbpl_status,
                                                 levels=TBPL_WORST_LEVEL_TUPLE)
 
         """
           We can run evaluate_parser multiple times, it will duplicate failures
           and status which can mean that future tests will fail if a previous test fails.
-          When we have a previous summary, we want to do 2 things:
-            1) Remove previous data from the new summary to only look at new data
-            2) Build a joined summary to include the previous + new data
+          When we have a previous summary, we want to do:
+            1) reset state so we only evaluate the current results
         """
-        keys = ['pass_count', 'fail_count', 'known_fail_count', 'crashed', 'leaked']
-        joined_summary = {}
-        for key in keys:
-            joined_summary[key] = getattr(self, key)
-
+        joined_summary = {'pass_count': self.pass_count}
         if previous_summary:
-            for key in keys:
-                joined_summary[key] += previous_summary[key]
-                value = getattr(self, key) - previous_summary[key]
-                if value < 0:
-                    value = 0
-                setattr(self, key, value)
             self.tbpl_status = TBPL_SUCCESS
             self.worst_log_level = INFO
+            self.crashed = False
+            self.leaked = False
 
         # I have to put this outside of parse_single_line because this checks not
         # only if fail_count was more then 0 but also if fail_count is still -1
         # (no fail summary line was found)
         if self.fail_count != 0:
             self.worst_log_level = self.worst_level(WARNING, self.worst_log_level)
             self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status,
                                                 levels=TBPL_WORST_LEVEL_TUPLE)
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -366,17 +366,18 @@ class WebPlatformTest(TestingMixin, Merc
                                                cwd=dirs['abs_work_dir'],
                                                output_timeout=1000,
                                                output_parser=parser,
                                                env=env)
 
                 if self.per_test_coverage:
                     self.add_per_test_coverage_report(gcov_dir, jsvm_dir, suite, per_test_args[-1])
 
-                tbpl_status, log_level, summary = parser.evaluate_parser(return_code, summary)
+                tbpl_status, log_level, summary = parser.evaluate_parser(return_code,
+                                                                         previous_summary=summary)
                 self.buildbot_status(tbpl_status, level=log_level)
 
                 if len(per_test_args) > 0:
                     self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
 
 
 # main {{{1
 if __name__ == '__main__':