Bug 1036325 - Support bisection for failing intermittent mochitests. r=jmaher
authorVaibhav Agrawal <vaibhavmagarwal@gmail.com>
Thu, 24 Jul 2014 07:54:00 +0200
changeset 196035 dc5dfa48e98d631c314ddfc353ef9207bf2aaeef
parent 196034 5f1dc690c1116abf69229d2fbe30d6a9c6fa0787
child 196036 38a505f2bcb10d7f551718da527c93a847a95c4d
push id46768
push usercbook@mozilla.com
push dateFri, 25 Jul 2014 07:38:08 +0000
treeherdermozilla-inbound@eed7bf748cbd [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1036325
milestone34.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1036325 - Support bisection for failing intermittent mochitests. r=jmaher
testing/mochitest/bisection.py
--- a/testing/mochitest/bisection.py
+++ b/testing/mochitest/bisection.py
@@ -6,16 +6,19 @@ class Bisect(object):
     "Class for creating, bisecting and summarizing for --bisect-chunk option."
 
     def __init__(self, harness):
         super(Bisect, self).__init__()
         self.summary = []
         self.contents = {}
         self.testRoot = harness.testRoot
         self.testRootAbs = harness.testRootAbs
+        self.repeat = 10
+        self.failcount = 0
+        self.max_failures = 3
 
     def setup(self, tests):
         "This method is used to initialize various variables that are required for test bisection"
         status = 0
         self.contents.clear()
         # We need totalTests key in contents for sanity check
         self.contents['totalTests'] = tests
         self.contents['tests'] = tests
@@ -198,23 +201,22 @@ class Bisect(object):
         if options.bisectChunk not in self.result:
             return -1
 
         self.summary.append("\tPass %d:" % self.contents['loop'])
         if len(self.contents['testsToRun']) > 1:
             self.summary.append("\t\t%d test files(start,end,failing). [%s, %s, %s]" % (len(self.contents['testsToRun']), self.contents['testsToRun'][0], self.contents['testsToRun'][-2], self.contents['testsToRun'][-1]))
         else:
             self.summary.append("\t\t1 test file [%s]" % self.contents['testsToRun'][0])
+            return self.check_for_intermittent(options)
 
         if self.result[options.bisectChunk] == "PASS":
             self.summary.append("\t\tno failures found.")
             if self.contents['loop'] == 1:
                 status = -1
-            elif self.contents['loop'] == 2:
-                status = 1
             else:
                 self.contents['result'] = "PASS"
                 status = 0
 
         elif self.result[options.bisectChunk] == "FAIL":
             if 'expectedError' not in self.contents:
                 self.summary.append("\t\t%s failed." % self.contents['testsToRun'][-1])
                 self.contents['expectedError'] = self.expectedError[options.bisectChunk]
@@ -232,13 +234,47 @@ class Bisect(object):
                     self.summary.append("TEST-BLEEDTHROUGH - found failure, %s" % self.contents['testsToRun'][0])
                     status = -1
             else:
                 self.summary.append("\t\t%s failed with different error." % self.contents['testsToRun'][-1])
                 status = -1
 
         return status
 
+    def check_for_intermittent(self, options):
+        "This method is used to check whether a test is an intermittent."
+        if self.result[options.bisectChunk] == "PASS":
+            self.summary.append("\t\tThe test %s passed." % self.contents['testsToRun'][0])
+            if self.repeat > 0:
+                # loop is set to 1 to again run the single test.
+                self.contents['loop'] = 1
+                self.repeat -= 1
+                return 0
+            else:
+                if self.failcount > 0:
+                    # -1 is being returned as the test is intermittent, so no need to bisect further.
+                    return -1
+                # If the test does not fail even once, then proceed to next chunk for bisection.
+                # loop is set to 2 to proceed on bisection.
+                self.contents['loop'] = 2
+                return 1
+        elif self.result[options.bisectChunk] == "FAIL":
+            self.summary.append("\t\tThe test %s failed." % self.contents['testsToRun'][0])
+            self.failcount += 1
+            self.contents['loop'] = 1
+            self.repeat -= 1
+            # self.max_failures is the maximum number of times a test is allowed
+            # to fail to be called an intermittent. If a test fails more than
+            # limit set, it is a perma-fail.
+            if self.failcount < self.max_failures:
+                if self.repeat == 0:
+                    # -1 is being returned as the test is intermittent, so no need to bisect further.
+                    return -1
+                return 0
+            else:
+                self.summary.append("TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the root cause for many of the above failures" % self.contents['testsToRun'][0])
+                return -1
+
     def print_summary(self):
         "This method is used to print the recorded summary."
         print "Bisection summary:"
         for line in self.summary:
             print line