Bug 1036374 - Adding a binary search algorithm for bisection of failing tests. r=jmaher
authorVaibhav Agrawal <vaibhavmagarwal@gmail.com>
Wed, 09 Jul 2014 13:20:00 +0200
changeset 193220 ab7e8e8c1e0002266998619f9b950293e406a003
parent 193219 5cbe17e565074fe9a0dcb036d68d42b5b7a61c43
child 193221 18ed7626a49b5ef1293d6b327967eadaf956ea04
push id46058
push usercbook@mozilla.com
push dateThu, 10 Jul 2014 08:26:30 +0000
treeherdermozilla-inbound@18ed7626a49b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1036374
milestone33.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1036374 - Adding a binary search algorithm for bisection of failing tests. r=jmaher
testing/mochitest/bisection.py
testing/mochitest/runtests.py
--- a/testing/mochitest/bisection.py
+++ b/testing/mochitest/bisection.py
@@ -81,17 +81,17 @@ class Bisect(object):
         "This method is used to call other methods for setting up variables and getting the list of tests for bisection."
         if options.bisectChunk == "default":
             return tests
         # The second condition in 'if' is required to verify that the failing test is the last one.
         elif 'loop' not in self.contents or not self.contents['tests'][-1].endswith(options.bisectChunk):
             tests = self.get_tests_for_bisection(options, tests)
             status = self.setup(tests)
 
-        return self.next_chunk_reverse(options, status)
+        return self.next_chunk_binary(options, status)
 
     def post_test(self, options, expectedError, result):
         "This method is used to call other methods to summarize results and check whether a sanity check is done or not."
         self.reset(expectedError, result)
         status = self.summarize_chunk(options)
         # Check whether sanity check has to be done. Also it is necessary to check whether options.bisectChunk is present
         # in self.expectedError as we do not want to run if it is "default".
         if status == -1 and options.bisectChunk in self.expectedError:
@@ -110,23 +110,21 @@ class Bisect(object):
             self.summary.append("Sanity Check:")
 
         return status
 
     def next_chunk_reverse(self, options, status):
         "This method is used to bisect the tests in a reverse search fashion."
 
         # Base Cases.
-        if self.contents['loop'] == 0:
-            self.contents['loop'] += 1
+        if self.contents['loop'] <= 1:
             self.contents['testsToRun'] = self.contents['tests']
-            return self.contents['testsToRun']
-        if self.contents['loop'] == 1:
+            if self.contents['loop'] == 1:
+                self.contents['testsToRun'] = [self.contents['tests'][-1]]
             self.contents['loop'] += 1
-            self.contents['testsToRun'] = [self.contents['tests'][-1]]
             return self.contents['testsToRun']
 
         if 'result' in self.contents:
             if self.contents['result'] == "PASS":
                 chunkSize = self.contents['end'] - self.contents['start']
                 self.contents['end'] = self.contents['start'] - 1
                 self.contents['start'] = self.contents['end'] - chunkSize
 
@@ -145,16 +143,52 @@ class Bisect(object):
         start = self.contents['start']
         end = self.contents['end'] + 1
         self.contents['testsToRun'] = self.contents['tests'][start:end]
         self.contents['testsToRun'].append(self.contents['tests'][-1])
         self.contents['loop'] += 1
 
         return self.contents['testsToRun']
 
+    def next_chunk_binary(self, options, status):
+        "This method is used to bisect the tests in a binary search fashion."
+
+        # Base cases.
+        if self.contents['loop'] <= 1:
+            self.contents['testsToRun'] = self.contents['tests']
+            if self.contents['loop'] == 1:
+                self.contents['testsToRun'] = [self.contents['tests'][-1]]
+            self.contents['loop'] += 1
+            return self.contents['testsToRun']
+
+        # Initialize the contents dict.
+        if status:
+            totalTests = len(self.contents['tests'])
+            self.contents['start'] = 0
+            self.contents['end'] = totalTests - 2
+
+        mid = (self.contents['start'] + self.contents['end']) / 2
+        if 'result' in self.contents:
+            if self.contents['result'] == "PASS":
+                self.contents['end'] = mid
+
+            elif self.contents['result'] == "FAIL":
+                self.contents['start'] = mid + 1
+
+        mid = (self.contents['start'] + self.contents['end']) / 2
+        start = mid + 1
+        end = self.contents['end'] + 1
+        self.contents['testsToRun'] = self.contents['tests'][start:end]
+        if not self.contents['testsToRun']:
+            self.contents['testsToRun'].append(self.contents['tests'][mid])
+        self.contents['testsToRun'].append(self.contents['tests'][-1])
+        self.contents['loop'] += 1
+
+        return self.contents['testsToRun']
+
     def summarize_chunk(self, options):
         "This method is used summarize the results after the list of tests is run."
         if options.bisectChunk == "default":
             # if no expectedError that means all the tests have successfully passed.
             if len(self.expectedError) == 0:
                 return -1
             options.bisectChunk = self.expectedError.keys()[0]
             self.summary.append("\tFound Error in test: %s" % options.bisectChunk)
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -1397,41 +1397,39 @@ class Mochitest(MochitestUtilsMixin):
     # Making an instance of bisect class for --bisect-chunk option.
     bisect = bisection.Bisect(self)
     finished = False
     status = 0
     while not finished:
       if options.bisectChunk:
         testsToRun = bisect.pre_test(options, testsToRun, status)
 
-      self.doTests(options, onLaunch, testsToRun)
+      result = self.doTests(options, onLaunch, testsToRun)
       if options.bisectChunk:
         status = bisect.post_test(options, self.expectedError, self.result)
       else:
         status = -1
 
       if status == -1:
         finished = True
 
     # We need to print the summary only if options.bisectChunk has a value.
     # Also we need to make sure that we do not print the summary in between running tests via --run-by-dir.
     if options.bisectChunk and options.bisectChunk in self.result:
       bisect.print_summary()
-      return -1
 
-    return 0
+    return result
 
   def runTests(self, options, onLaunch=None):
     """ Prepare, configure, run tests and cleanup """
 
     self.setTestRoot(options)
 
     if not options.runByDir:
-      self.runMochitests(options, onLaunch)
-      return 0
+      return self.runMochitests(options, onLaunch)
 
     # code for --run-by-dir
     dirs = self.getDirectories(options)
 
     if options.totalChunks > 1:
       chunkSize = int(len(dirs) / options.totalChunks) + 1
       start = chunkSize * (options.thisChunk-1)
       end = chunkSize * (options.thisChunk)
@@ -1446,35 +1444,35 @@ class Mochitest(MochitestUtilsMixin):
         continue
 
       options.testPath = dir
       print "testpath: %s" % options.testPath
 
       # If we are using --run-by-dir, we should not use the profile path (if) provided
       # by the user, since we need to create a new directory for each run. We would face problems
       # if we use the directory provided by the user.
-      runResult = self.runMochitests(options, onLaunch)
-      if runResult == -1:
-        return 0
+      result = self.runMochitests(options, onLaunch)
 
     # printing total number of tests
     if options.browserChrome:
       print "TEST-INFO | checking window state"
       print "Browser Chrome Test Summary"
       print "\tPassed: %s" % self.countpass
       print "\tFailed: %s" % self.countfail
       print "\tTodo: %s" % self.counttodo
       print "*** End BrowserChrome Test Results ***"
     else:
       print "0 INFO TEST-START | Shutdown"
       print "1 INFO Passed:  %s" % self.countpass
       print "2 INFO Failed:  %s" % self.countfail
       print "3 INFO Todo:    %s" % self.counttodo
       print "4 INFO SimpleTest FINISHED"
 
+    return result
+
   def doTests(self, options, onLaunch=None, testsToFilter = None):
     # A call to initializeLooping method is required in case of --run-by-dir or --bisect-chunk
     # since we need to initialize variables for each loop.
     if options.bisectChunk or options.runByDir:
       self.initializeLooping(options)
 
     # get debugger info, a dict of:
     # {'path': path to the debugger (string),