Bug 1445716 - Process more geckoview-junit status codes; r=jmaher
authorGeoff Brown <gbrown@mozilla.com>
Wed, 25 Apr 2018 09:10:26 -0600
changeset 471719 1b29fa5bb80221c4ef0b70851de5c0b4bad078a6
parent 471718 2197be608baf332f861a4a5a5fc75a6b9dca3786
child 471720 dbcb5f8ea056f2307b867e9ceafa4772b91c2a00
push id1728
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:12:27 +0000
treeherdermozilla-release@c296fde26f5f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1445716
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1445716 - Process more geckoview-junit status codes; r=jmaher
testing/mochitest/runjunit.py
--- a/testing/mochitest/runjunit.py
+++ b/testing/mochitest/runjunit.py
@@ -186,26 +186,26 @@ class JUnitTestRunner(MochitestDesktop):
                             self.options.app)
         if self.device.process_exist(self.options.app):
             raise Exception("%s already running before starting tests" %
                             self.options.app)
 
         self.test_started = False
         self.pass_count = 0
         self.fail_count = 0
+        self.todo_count = 0
         self.class_name = ""
         self.test_name = ""
         self.current_full_name = ""
 
         def callback(line):
             # Output callback: Parse the raw junit log messages, translating into
             # treeherder-friendly test start/pass/fail messages.
 
             self.log.process_output(self.options.app, str(line))
-            expected = 'PASS'
             # Expect per-test info like: "INSTRUMENTATION_STATUS: class=something"
             match = re.match(r'INSTRUMENTATION_STATUS:\s*class=(.*)', line)
             if match:
                 self.class_name = match.group(1)
             # Expect per-test info like: "INSTRUMENTATION_STATUS: test=something"
             match = re.match(r'INSTRUMENTATION_STATUS:\s*test=(.*)', line)
             if match:
                 self.test_name = match.group(1)
@@ -213,42 +213,57 @@ class JUnitTestRunner(MochitestDesktop):
             match = re.match(r'INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)', line)
             if match:
                 status = match.group(1)
                 full_name = "%s.%s" % (self.class_name, self.test_name)
                 if full_name == self.current_full_name:
                     if status == '0':
                         message = ''
                         status = 'PASS'
+                        expected = 'PASS'
                         self.pass_count += 1
+                    elif status == '-3':  # ignored (skipped)
+                        message = ''
+                        status = 'SKIP'
+                        expected = 'SKIP'
+                        self.todo_count += 1
+                    elif status == '-4':  # known fail
+                        message = ''
+                        status = 'FAIL'
+                        expected = 'FAIL'
+                        self.todo_count += 1
                     else:
                         message = 'status %s' % status
                         status = 'FAIL'
+                        expected = 'PASS'
                         self.fail_count += 1
                     self.log.test_end(full_name, status, expected, message)
                     self.test_started = False
                 else:
                     if self.test_started:
                         # next test started without reporting previous status
                         self.fail_count += 1
-                        self.log.test_end(self.current_full_name, 'FAIL', expected,
+                        status = 'FAIL'
+                        expected = 'PASS'
+                        self.log.test_end(self.current_full_name, status, expected,
                                           "missing test completion status")
                     self.log.test_start(full_name)
                     self.test_started = True
                     self.current_full_name = full_name
 
         # Ideally all test names should be reported to suite_start, but these test
         # names are not known in advance.
         self.log.suite_start(["geckoview-junit"])
         try:
             cmd = self.build_command_line(test_filters)
             self.log.info("launching %s" % cmd)
             self.device.shell(cmd, timeout=self.options.max_time, stdout_callback=callback)
             self.log.info("Passed: %d" % self.pass_count)
             self.log.info("Failed: %d" % self.fail_count)
+            self.log.info("Todo: %d" % self.todo_count)
         finally:
             self.log.suite_end()
 
         if self.check_for_crashes():
             self.fail_count = 1
 
         return 1 if self.fail_count else 0