Bug 916797 - --run-until-failure mochitest option should support running multiple tests. r=jmaher
authorFelipe Gomes <felipc@gmail.com>
Tue, 05 Nov 2013 13:48:36 -0200
changeset 153600 34307fd1289e8fd02567124e07a26bd9a22a2d9a
parent 153599 eb5d47884a27eb2817d4b4ae8de842a41a6de8cf
child 153601 f6a8fcd4a4475e00eba2c58800849cd02da67e0a
push id25595
push userryanvm@gmail.com
push dateTue, 05 Nov 2013 20:19:27 +0000
treeherdermozilla-central@2ada3a06d5e7 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs916797
milestone28.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 916797 - --run-until-failure mochitest option should support running multiple tests. r=jmaher
testing/mochitest/browser-test.js
testing/mochitest/mach_commands.py
testing/mochitest/mochitest_options.py
testing/mochitest/tests/SimpleTest/TestRunner.js
--- a/testing/mochitest/browser-test.js
+++ b/testing/mochitest/browser-test.js
@@ -180,19 +180,16 @@ Tester.prototype = {
     this.SimpleTest.waitForFocus(aCallback);
   },
 
   finish: function Tester_finish(aSkipSummary) {
     var passCount = this.tests.reduce(function(a, f) a + f.passCount, 0);
     var failCount = this.tests.reduce(function(a, f) a + f.failCount, 0);
     var todoCount = this.tests.reduce(function(a, f) a + f.todoCount, 0);
 
-    if (failCount > 0 && this.runUntilFailure)
-      this.repeat = 0;
-
     if (this.repeat > 0) {
       --this.repeat;
       this.currentTestIndex = -1;
       this.nextTest();
     }
     else{
       Services.console.unregisterListener(this);
       Services.obs.removeObserver(this, "chrome-document-global-created");
@@ -217,16 +214,22 @@ Tester.prototype = {
       // Tests complete, notify the callback and return
       this.callback(this.tests);
       this.callback = null;
       this.tests = null;
       this.openedWindows = null;
     }
   },
 
+  haltTests: function Tester_haltTests() {
+    // Do not run any further tests
+    this.currentTestIndex = this.tests.length - 1;
+    this.repeat = 0;
+  },
+
   observe: function Tester_observe(aSubject, aTopic, aData) {
     if (!aTopic) {
       this.onConsoleMessage(aSubject);
     } else if (this.currentTest) {
       this.onDocumentCreated(aSubject);
     }
   },
 
@@ -349,16 +352,20 @@ Tester.prototype = {
         }
       }
 
       // Note the test run time
       let time = Date.now() - this.lastStartTime;
       this.dumper.dump("INFO TEST-END | " + this.currentTest.path + " | finished in " + time + "ms\n");
       this.currentTest.setDuration(time);
 
+      if (this.runUntilFailure && this.currentTest.failCount > 0) {
+        this.haltTests();
+      }
+
       testScope.destroy();
       this.currentTest.scope = null;
     }
 
     // Check the window state for the current test before moving to the next one.
     // This also causes us to check before starting any tests, since nextTest()
     // is invoked to start the tests.
     this.waitForWindowsState((function () {
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -387,19 +387,19 @@ def MochitestCommand(func):
         help='Do not starting running tests automatically.')
     func = autorun(func)
 
     repeat = CommandArgument('--repeat', type=int, default=0,
         help='Repeat the test the given number of times.')
     func = repeat(func)
 
     runUntilFailure = CommandArgument("--run-until-failure", action='store_true',
-        help='Run a test repeatedly and stops on the first time the test fails. ' \
-             'Only available when running a single test. Default cap is 30 runs, ' \
-             'which can be overwritten with the --repeat parameter.')
+        help='Run tests repeatedly and stops on the first time a test fails. ' \
+             'Default cap is 30 runs, which can be overwritten ' \
+             'with the --repeat parameter.')
     func = runUntilFailure(func)
 
     slow = CommandArgument('--slow', action='store_true',
         help='Delay execution between tests.')
     func = slow(func)
 
     end_at = CommandArgument('--end-at', type=str,
         help='Stop running the test sequence at this test.')
--- a/testing/mochitest/mochitest_options.py
+++ b/testing/mochitest/mochitest_options.py
@@ -269,19 +269,18 @@ class MochitestOptions(optparse.OptionPa
           "dest": "repeat",
           "metavar": "REPEAT",
           "help": "repeats the test or set of tests the given number of times, ie: repeat: 1 will run the test twice.",
           "default": 0,
         }],
         [["--run-until-failure"],
         { "action": "store_true",
           "dest": "runUntilFailure",
-          "help": "Run a test repeatedly and stops on the first time the test fails. "
-                "Only available when running a single test. Default cap is 30 runs, "
-                "which can be overwritten with the --repeat parameter.",
+          "help": "Run tests repeatedly and stops on the first time a test fails. "
+                "Default cap is 30 runs, which can be overwritten with the --repeat parameter.",
           "default": False,
         }],
         [["--run-only-tests"],
         { "action": "store",
           "type": "string",
           "dest": "runOnlyTests",
           "help": "JSON list of tests that we only want to run. [DEPRECATED- please use --test-manifest]",
           "default": None,
@@ -490,18 +489,16 @@ class MochitestOptions(optparse.OptionPa
                 self.error("immersive is only supported on Windows 8 and up.")
             mochitest.immersiveHelperPath = os.path.join(
                 options.utilityPath, "metrotestharness.exe")
             if not os.path.exists(mochitest.immersiveHelperPath):
                 self.error("%s not found, cannot launch immersive tests." %
                            mochitest.immersiveHelperPath)
 
         if options.runUntilFailure:
-            if not os.path.isfile(os.path.join(mochitest.oldcwd, os.path.dirname(__file__), mochitest.getTestRoot(options), options.testPath)):
-                self.error("--run-until-failure can only be used together with --test-path specifying a single test.")
             if not options.repeat:
                 options.repeat = 29
 
         return options
 
 
 class B2GOptions(MochitestOptions):
     b2g_options = [
--- a/testing/mochitest/tests/SimpleTest/TestRunner.js
+++ b/testing/mochitest/tests/SimpleTest/TestRunner.js
@@ -209,16 +209,20 @@ TestRunner.log = function(msg) {
 
 TestRunner.error = function(msg) {
     if (TestRunner.logEnabled) {
         TestRunner.logger.error(msg);
     } else {
         dump(msg + "\n");
     }
 
+    if (TestRunner.runUntilFailure) {
+      TestRunner._haltTests = true;
+    }
+
     if (TestRunner.debugOnFailure) {
       // You've hit this line because you requested to break into the
       // debugger upon a testcase failure on your test run.
       debugger;
     }
 };
 
 /**
@@ -361,20 +365,17 @@ TestRunner.runNextTest = function() {
         if (TestRunner.repeat == 0) {
           TestRunner.log("SimpleTest FINISHED");
         }
 
         if (TestRunner.repeat == 0 && TestRunner.onComplete) {
              TestRunner.onComplete();
          }
 
-        var failCount = parseInt($("fail-count").innerHTML);
-        var stopLooping = failCount > 0 && TestRunner.runUntilFailure;
-
-        if (TestRunner._currentLoop <= TestRunner.repeat && !stopLooping) {
+        if (TestRunner._currentLoop <= TestRunner.repeat && !TestRunner._haltTests) {
           TestRunner._currentLoop++;
           TestRunner.resetTests(TestRunner._urls);
           TestRunner._loopIsRestarting = true;
         } else {
           // Loops are finished
           if (TestRunner.logEnabled) {
             TestRunner.log("TEST-INFO | Ran " + TestRunner._currentLoop + " Loops");
             TestRunner.log("SimpleTest FINISHED");