Bug 725112 - Add an easy make command to rerun tests that failed the last time. r=ted
authorJoel Maher <jmaher@mozilla.com>
Mon, 20 Feb 2012 09:59:04 -0500
changeset 87227 0dbfda1a8f0c4f9d4f352bf02ecdc530f6e6bc82
parent 87226 81c166bac966d0b225b78220264cbef5256266c5
child 87228 13ff3e5b66eae8f612b9f8804d560c985f34b216
push id22103
push userbmo@edmorley.co.uk
push dateTue, 21 Feb 2012 12:01:45 +0000
treeherdermozilla-central@4038ffaa5d82 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs725112
milestone13.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 725112 - Add an easy make command to rerun tests that failed the last time. r=ted
testing/mochitest/runtests.py
testing/mochitest/tests/SimpleTest/SimpleTest.js
testing/mochitest/tests/SimpleTest/TestRunner.js
testing/mochitest/tests/SimpleTest/setup.js
testing/testsuite-targets.mk
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -243,16 +243,21 @@ class MochitestOptions(optparse.OptionPa
                     help = "JSON list of tests that we only want to run, cannot be specified with --exclude-tests.")
     defaults["runOnlyTests"] = None
 
     self.add_option("--exclude-tests",
                     action = "store", type="string", dest = "excludeTests",
                     help = "JSON list of tests that we want to not run, cannot be specified with --run-only-tests.")
     defaults["excludeTests"] = None
 
+    self.add_option("--failure-file",
+                    action = "store", type="string", dest = "failureFile",
+                    help = "Filename of the output file where we can store a .json list of failures to be run in the future with --run-only-tests.")
+    defaults["failureFile"] = None
+
     # -h, --help are automatically handled by OptionParser
 
     self.set_defaults(**defaults)
 
     usage = """\
 Usage instructions for runtests.py.
 All arguments are optional.
 If --chrome is specified, chrome tests will be run instead of web content tests.
@@ -309,23 +314,23 @@ See <http://mochikit.com/doc/html/MochiK
         options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll")
       if not os.path.exists(mochitest.vmwareHelperPath):
         self.error("%s not found, cannot automate VMware recording." %
                    mochitest.vmwareHelperPath)
 
     if options.runOnlyTests != None and options.excludeTests != None:
       self.error("We can only support --run-only-tests OR --exclude-tests, not both.")
       
-    if (options.runOnlyTests):
-      if (not os.path.exists(os.path.join(os.path.dirname(__file__), options.runOnlyTests))):
-        self.error("unable to find --run-only-tests file '%s'" % (options.runOnlyTests));
+    if options.runOnlyTests:
+      if not os.path.exists(os.path.abspath(options.runOnlyTests)):
+        self.error("unable to find --run-only-tests file '%s'" % options.runOnlyTests);
         
-    if (options.excludeTests):
-      if (not os.path.exists(os.path.join(os.path.dirname(__file__), options.excludeTests))):
-        self.error("unable to find --exclude-tests file '%s'" % (options.excludeTests));
+    if options.excludeTests:
+      if not os.path.exists(os.path.abspath(options.excludeTests)):
+        self.error("unable to find --exclude-tests file '%s'" % options.excludeTests);
 
     return options
 
 
 #######################
 # HTTP SERVER SUPPORT #
 #######################
 
@@ -589,16 +594,18 @@ class Mochitest(object):
       if "MOZ_HIDE_RESULTS_TABLE" in env and env["MOZ_HIDE_RESULTS_TABLE"] == "1":
         self.urlOpts.append("hideResultsTable=1")
       if options.repeat:
         self.urlOpts.append("repeat=%d" % options.repeat)
       if os.path.isfile(os.path.join(self.oldcwd, os.path.dirname(__file__), self.TEST_PATH, options.testPath)) and options.repeat > 0:
         self.urlOpts.append("testname=%s" % ("/").join([self.TEST_PATH, options.testPath]))
       if options.runOnlyTests:
         self.urlOpts.append("runOnlyTests=%s" % options.runOnlyTests)
+      if options.failureFile:
+        self.urlOpts.append("failureFile=%s" % options.failureFile)
       elif options.excludeTests:
         self.urlOpts.append("excludeTests=%s" % options.excludeTests)
 
   def cleanup(self, manifest, options):
     """ remove temporary files and profile """
     os.remove(manifest)
     shutil.rmtree(options.profilePath)
 
--- a/testing/mochitest/tests/SimpleTest/SimpleTest.js
+++ b/testing/mochitest/tests/SimpleTest/SimpleTest.js
@@ -269,16 +269,17 @@ SimpleTest._getCurrentTestURL = function
 SimpleTest._logResult = function(test, passString, failString) {
     var isError = !test.result == !test.todo;
     var resultString = test.result ? passString : failString;
     var url = SimpleTest._getCurrentTestURL();
     var diagnostic = test.name + (test.diag ? " - " + test.diag : "");
     var msg = [resultString, url, diagnostic].join(" | ");
     if (parentRunner) {
         if (isError) {
+            parentRunner.addFailedTest(url);
             parentRunner.error(msg);
         } else {
             parentRunner.log(msg);
         }
     } else {
         dump(msg + "\n");
     }
 };
--- a/testing/mochitest/tests/SimpleTest/TestRunner.js
+++ b/testing/mochitest/tests/SimpleTest/TestRunner.js
@@ -128,16 +128,38 @@ TestRunner.repeat = 0;
 TestRunner._currentLoop = 0;
 
 /**
  * This function is called after generating the summary.
 **/
 TestRunner.onComplete = null;
 
 /**
+ * Adds a failed test case to a list so we can rerun only the failed tests
+ **/
+TestRunner._failedTests = {};
+TestRunner._failureFile = "";
+
+TestRunner.addFailedTest = function(testName) {
+    if (TestRunner._failedTests[testName] == undefined) {
+        TestRunner._failedTests[testName] = "";
+    }
+};
+
+TestRunner.setFailureFile = function(fileName) {
+    TestRunner._failureFile = fileName;
+}
+
+TestRunner.generateFailureList = function() {
+    var failures = new SpecialPowersLogger(TestRunner._failureFile);
+    failures.log(JSON.stringify(TestRunner._failedTests));
+    failures.close();
+};
+
+/**
  * If logEnabled is true, this is the logger that will be used.
 **/
 TestRunner.logger = LogController;
 
 TestRunner.log = function(msg) {
     if (TestRunner.logEnabled) {
         TestRunner.logger.log(msg);
     } else {
@@ -341,16 +363,17 @@ TestRunner.runNextTest = function() {
           if (TestRunner.logEnabled) {
             TestRunner.log("TEST-INFO | Ran " + TestRunner._currentLoop + " Loops");
             TestRunner.log("SimpleTest FINISHED");
           }
 
           if (TestRunner.onComplete)
             TestRunner.onComplete();
        }
+       TestRunner.generateFailureList();
     }
 };
 
 TestRunner.expectChildProcessCrash = function() {
     TestRunner._expectingProcessCrash = true;
 };
 
 /**
--- a/testing/mochitest/tests/SimpleTest/setup.js
+++ b/testing/mochitest/tests/SimpleTest/setup.js
@@ -113,16 +113,20 @@ if (params.repeat) {
   TestRunner.repeat = params.repeat;
 } 
 
 // closeWhenDone tells us to close the browser when complete
 if (params.closeWhenDone) {
   TestRunner.onComplete = SpecialPowers.quit;
 }
 
+if (params.failureFile) {
+  TestRunner.setFailureFile(params.failureFile);
+}
+
 // logFile to write our results
 if (params.logFile) {
   var spl = new SpecialPowersLogger(params.logFile);
   TestRunner.logger.addListener("mozLogger", fileLevel + "", spl.getLogCallback());
 }
 
 // if we get a quiet param, don't log to the console
 if (!params.quiet) {
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -60,16 +60,24 @@ mochitest:: $(MOCHITESTS)
 ifndef TEST_PACKAGE_NAME
 TEST_PACKAGE_NAME := $(ANDROID_PACKAGE_NAME)
 endif
 
 RUN_MOCHITEST = \
 	rm -f ./$@.log && \
 	$(PYTHON) _tests/testing/mochitest/runtests.py --autorun --close-when-done \
 	  --console-level=INFO --log-file=./$@.log --file-level=INFO \
+	  --failure-file=$(call core_abspath,_tests/testing/mochitest/makefailures.json)  \
+	  $(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS)
+
+RERUN_MOCHITEST = \
+	rm -f ./$@.log && \
+	$(PYTHON) _tests/testing/mochitest/runtests.py --autorun --close-when-done \
+	  --console-level=INFO --log-file=./$@.log --file-level=INFO \
+	  --run-only-tests=makefailures.json  \
 	  $(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS)
 
 RUN_MOCHITEST_REMOTE = \
 	rm -f ./$@.log && \
 	$(PYTHON) _tests/testing/mochitest/runtestsremote.py --autorun --close-when-done \
 	  --console-level=INFO --log-file=./$@.log --file-level=INFO $(DM_FLAGS) --dm_trans=$(DM_TRANS) \
 	  --app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \
 	  $(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS)
@@ -83,16 +91,17 @@ RUN_MOCHITEST_ROBOTIUM = \
     --robocop=$(DEPTH)/build/mobile/robocop/robocop.ini $(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS)
 
 ifndef NO_FAIL_ON_TEST_ERRORS
 define CHECK_TEST_ERROR
   @errors=`grep "TEST-UNEXPECTED-" $@.log` ;\
   if test "$$errors" ; then \
 	  echo "$@ failed:"; \
 	  echo "$$errors"; \
+          echo "To rerun your failures please run 'make mochitest-plain-rerun-failures'"; \
 	  exit 1; \
   else \
 	  echo "$@ passed"; \
   fi
 endef
 endif
 
 mochitest-remote: DM_TRANS?=adb
@@ -115,16 +124,20 @@ mochitest-robotium:
     else \
         $(RUN_MOCHITEST_ROBOTIUM); \
     fi
 
 mochitest-plain:
 	$(RUN_MOCHITEST)
 	$(CHECK_TEST_ERROR)
 
+mochitest-plain-rerun-failures:
+	$(RERUN_MOCHITEST)
+	$(CHECK_TEST_ERROR)
+
 # Allow mochitest-1 ... mochitest-5 for developer ease
 mochitest-1 mochitest-2 mochitest-3 mochitest-4 mochitest-5: mochitest-%:
 	echo "mochitest: $* / 5"
 	$(RUN_MOCHITEST) --chunk-by-dir=4 --total-chunks=5 --this-chunk=$*
 	$(CHECK_TEST_ERROR)
 
 mochitest-chrome:
 	$(RUN_MOCHITEST) --chrome