Bug 1395935 - Include leaks in mochitest summary fail count; r=ahal
authorGeoff Brown <gbrown@mozilla.com>
Thu, 23 Nov 2017 10:02:22 -0700
changeset 437879 f6d80e35ce4f704e73c9073b2e56939dcde8c794
parent 437878 0ee02ed83f71b4c4d2ed2d142eea26c5ff15081c
child 437880 72f457e76554b822f4b777706360d7a54de11a83
push id117
push userfmarier@mozilla.com
push dateTue, 28 Nov 2017 20:17:16 +0000
reviewersahal
bugs1395935
milestone59.0a1
Bug 1395935 - Include leaks in mochitest summary fail count; r=ahal
testing/mochitest/leaks.py
testing/mochitest/runtests.py
--- a/testing/mochitest/leaks.py
+++ b/testing/mochitest/leaks.py
@@ -46,39 +46,46 @@ class ShutdownLeaks(object):
                 "fileName": fileName, "windows": set(), "docShells": set()}
         elif action == 'test_end':
             # don't track a test if no windows or docShells leaked
             if self.currentTest and (self.currentTest["windows"] or self.currentTest["docShells"]):
                 self.tests.append(self.currentTest)
             self.currentTest = None
 
     def process(self):
+        failures = 0
+
         if not self.seenShutdown:
             self.logger.error(
                 "TEST-UNEXPECTED-FAIL | ShutdownLeaks | process() called before end of test suite")
+            failures += 1
 
         for test in self._parseLeakingTests():
             for url, count in self._zipLeakedWindows(test["leakedWindows"]):
                 self.logger.error(
                     "TEST-UNEXPECTED-FAIL | %s | leaked %d window(s) until shutdown "
                     "[url = %s]" % (test["fileName"], count, url))
+                failures += 1
 
             if test["leakedWindowsString"]:
                 self.logger.info("TEST-INFO | %s | windows(s) leaked: %s" %
                                  (test["fileName"], test["leakedWindowsString"]))
 
             if test["leakedDocShells"]:
                 self.logger.error("TEST-UNEXPECTED-FAIL | %s | leaked %d docShell(s) until "
                                   "shutdown" %
                                   (test["fileName"], len(test["leakedDocShells"])))
+                failures += 1
                 self.logger.info("TEST-INFO | %s | docShell(s) leaked: %s" %
                                  (test["fileName"], ', '.join(["[pid = %s] [id = %s]" %
                                                                x for x in test["leakedDocShells"]]
                                                               )))
 
+        return failures
+
     def _logWindow(self, line):
         created = line[:2] == "++"
         pid = self._parseValue(line, "pid")
         serial = self._parseValue(line, "serial")
 
         # log line has invalid format
         if not pid or not serial:
             self.logger.error(
@@ -238,35 +245,42 @@ class LSANLeaks(object):
             # System library stack frames will never match the skip list,
             # so don't bother checking if they do.
             self._recordFrame(sysLibStackFrame.group(1))
 
         # If we don't match either of these, just ignore the frame.
         # We'll end up with "unknown stack" if everything is ignored.
 
     def process(self):
+        failures = 0
+
         if self.fatalError:
             self.logger.error("TEST-UNEXPECTED-FAIL | LeakSanitizer | LeakSanitizer "
                               "has encountered a fatal error.")
+            failures += 1
 
         if self.symbolizerError:
             self.logger.error("TEST-UNEXPECTED-FAIL | LeakSanitizer | LLVMSymbolizer "
                               "was unable to allocate memory.")
+            failures += 1
             self.logger.info("TEST-INFO | LeakSanitizer | This will cause leaks that "
                              "should be ignored to instead be reported as an error")
 
         if self.foundFrames:
             self.logger.info("TEST-INFO | LeakSanitizer | To show the "
                              "addresses of leaked objects add report_objects=1 to LSAN_OPTIONS")
             self.logger.info("TEST-INFO | LeakSanitizer | This can be done "
                              "in testing/mozbase/mozrunner/mozrunner/utils.py")
 
         for f in self.foundFrames:
             self.logger.error(
                 "TEST-UNEXPECTED-FAIL | LeakSanitizer | leak at " + f)
+            failures += 1
+
+        return failures
 
     def _finishStack(self):
         if self.recordMoreFrames and len(self.currStack) == 0:
             self.currStack = ["unknown stack"]
         if self.currStack:
             self.foundFrames.add(", ".join(self.currStack))
             self.currStack = None
         self.recordMoreFrames = False
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -2925,20 +2925,20 @@ toolbar#nav-bar {
         def stackFixer(self):
             """
             return get_stack_fixer_function, if any, to use on the output lines
             """
             return get_stack_fixer_function(self.utilityPath, self.symbolsPath)
 
         def finish(self):
             if self.shutdownLeaks:
-                self.shutdownLeaks.process()
+                self.harness.countfail += self.shutdownLeaks.process()
 
             if self.lsanLeaks:
-                self.lsanLeaks.process()
+                self.harness.countfail += self.lsanLeaks.process()
 
         # output message handlers:
         # these take a message and return a message
 
         def record_result(self, message):
             # by default make the result key equal to pass.
             if message['action'] == 'test_start':
                 key = message['test'].split('/')[-1].strip()