Bug 404077: Add mochitest support (currently semi-disabled) for making tests fail when an unexpected number of assertions fire. r=ted
authorL. David Baron <dbaron@dbaron.org>
Sun, 24 Feb 2013 23:42:38 -0800
changeset 122889 d1490171893be5c3d8c9328fbd2da8de4cf8512e
parent 122888 fb5a9401e2d4bf5be9fb9d3a96a9077b55dd8f81
child 122890 7e5295f718b3117853196adead2caa3e08ae1f73
push id1387
push userphilringnalda@gmail.com
push dateTue, 26 Feb 2013 22:32:56 +0000
treeherderfx-team@ad4cc4e97774 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs404077
milestone22.0a1
Bug 404077: Add mochitest support (currently semi-disabled) for making tests fail when an unexpected number of assertions fire. r=ted This adds support for assertion checking in all mochitest suites except for mochitest-browser-chrome. The checking works much like it does in reftest, except for the mechanism for annotating expected assertions, SimpleTest.expectAssertions() (see its in-code documentation). The support is initially disabled in that: (1) It doesn't cause the tests to report failure (and thus turn the tree orange). (2) It prints TEST-DETCEPXENU-FAIL/PASS instead of TEST-UNEXPECTED-FAIL/PASS (so that it doesn't show up in log highlighting). The assertion checking only works within the test runner (which runs multiple tests); it does not function when running only a single test.
testing/mochitest/jar.mn
testing/mochitest/tests/SimpleTest/Makefile.in
testing/mochitest/tests/SimpleTest/SimpleTest.js
testing/mochitest/tests/SimpleTest/TestRunner.js
testing/mochitest/tests/SimpleTest/iframe-between-tests.html
testing/specialpowers/content/specialpowersAPI.js
--- a/testing/mochitest/jar.mn
+++ b/testing/mochitest/jar.mn
@@ -17,14 +17,15 @@ mochikit.jar:
   content/tests/SimpleTest/LogController.js (tests/SimpleTest/LogController.js)
   content/tests/SimpleTest/MozillaLogger.js (../specialpowers/content/MozillaLogger.js)
   content/tests/SimpleTest/SpecialPowersObserverAPI.js (../specialpowers/content/SpecialPowersObserverAPI.js)
   content/tests/SimpleTest/specialpowersAPI.js (../specialpowers/content/specialpowersAPI.js)
   content/tests/SimpleTest/setup.js (tests/SimpleTest/setup.js)
   content/tests/SimpleTest/SimpleTest.js (tests/SimpleTest/SimpleTest.js)
   content/tests/SimpleTest/test.css (tests/SimpleTest/test.css)
   content/tests/SimpleTest/TestRunner.js (tests/SimpleTest/TestRunner.js)
+  content/tests/SimpleTest/iframe-between-tests.html (tests/SimpleTest/iframe-between-tests.html)
   content/tests/SimpleTest/WindowSnapshot.js (tests/SimpleTest/WindowSnapshot.js)
   content/tests/SimpleTest/MockObjects.js (tests/SimpleTest/MockObjects.js)
   content/tests/SimpleTest/NativeKeyCodes.js (tests/SimpleTest/NativeKeyCodes.js)
   content/tests/SimpleTest/paint_listener.js (tests/SimpleTest/paint_listener.js)
   content/tests/SimpleTest/docshell_helpers.js (../../docshell/test/chrome/docshell_helpers.js)
 
--- a/testing/mochitest/tests/SimpleTest/Makefile.in
+++ b/testing/mochitest/tests/SimpleTest/Makefile.in
@@ -16,15 +16,16 @@ include $(topsrcdir)/config/rules.mk
 			TestRunner.js \
 			setup.js \
 			EventUtils.js \
 			ChromeUtils.js \
 			WindowSnapshot.js \
 			MockObjects.js \
 			NativeKeyCodes.js \
 			paint_listener.js \
+			iframe-between-tests.html \
 			$(DEPTH)/testing/specialpowers/content/MozillaLogger.js \
 			$(DEPTH)/docshell/test/chrome/docshell_helpers.js \
 			$(NULL)
 
 libs:: $(_SIMPLETEST_FILES)
 	$(INSTALL) $(foreach f,$^,"$f") $(DEPTH)/_tests/$(relativesrcdir)
 
--- a/testing/mochitest/tests/SimpleTest/SimpleTest.js
+++ b/testing/mochitest/tests/SimpleTest/SimpleTest.js
@@ -463,16 +463,54 @@ SimpleTest.waitForExplicitFinish = funct
  * finish.
  */
 SimpleTest.requestLongerTimeout = function (factor) {
     if (parentRunner) {
         parentRunner.requestLongerTimeout(factor);
     }
 }
 
+/**
+ * Note that the given range of assertions is to be expected.  When
+ * this function is not called, 0 assertions are expected.  When only
+ * one argument is given, that number of assertions are expected.
+ *
+ * A test where we expect to have assertions (which should largely be a
+ * transitional mechanism to get assertion counts down from our current
+ * situation) can call the SimpleTest.expectAssertions() function, with
+ * either one or two arguments:  one argument gives an exact number
+ * expected, and two arguments give a range.  For example, a test might do
+ * one of the following:
+ *
+ *   // Currently triggers two assertions (bug NNNNNN).
+ *   SimpleTest.expectAssertions(2);
+ *
+ *   // Currently triggers one assertion on Mac (bug NNNNNN).
+ *   if (navigator.platform.indexOf("Mac") == 0) {
+ *     SimpleTest.expectAssertions(1);
+ *   }
+ *
+ *   // Currently triggers two assertions on all platforms (bug NNNNNN),
+ *   // but intermittently triggers two additional assertions (bug NNNNNN)
+ *   // on Windows.
+ *   if (navigator.platform.indexOf("Win") == 0) {
+ *     SimpleTest.expectAssertions(2, 4);
+ *   } else {
+ *     SimpleTest.expectAssertions(2);
+ *   }
+ *
+ *   // Intermittently triggers up to three assertions (bug NNNNNN).
+ *   SimpleTest.expectAssertions(0, 3);
+ */
+SimpleTest.expectAssertions = function(min, max) {
+    if (parentRunner) {
+        parentRunner.expectAssertions(min, max);
+    }
+}
+
 SimpleTest.waitForFocus_started = false;
 SimpleTest.waitForFocus_loaded = false;
 SimpleTest.waitForFocus_focused = false;
 SimpleTest._pendingWaitForFocusCount = 0;
 
 /**
  * If the page is not yet loaded, waits for the load event. In addition, if
  * the page is not yet focused, focuses and waits for the window to be
--- a/testing/mochitest/tests/SimpleTest/TestRunner.js
+++ b/testing/mochitest/tests/SimpleTest/TestRunner.js
@@ -67,19 +67,23 @@ function flattenArguments(lst/* ...*/) {
  *
  *  * Avoid moving iframes: That causes reloads on mozilla and opera.
  *
  *
 **/
 var TestRunner = {};
 TestRunner.logEnabled = false;
 TestRunner._currentTest = 0;
+TestRunner._lastTestFinished = -1;
 TestRunner.currentTestURL = "";
 TestRunner.originalTestURL = "";
 TestRunner._urls = [];
+TestRunner._lastAssertionCount = 0;
+TestRunner._expectedMinAsserts = 0;
+TestRunner._expectedMaxAsserts = 0;
 
 TestRunner.timeout = 5 * 60 * 1000; // 5 minutes.
 TestRunner.maxTimeouts = 4; // halt testing after too many timeouts
 TestRunner.runSlower = false;
 
 TestRunner._expectingProcessCrash = false;
 
 /**
@@ -141,16 +145,28 @@ TestRunner.requestLongerTimeout = functi
 }
 
 /**
  * This is used to loop tests
 **/
 TestRunner.repeat = 0;
 TestRunner._currentLoop = 0;
 
+TestRunner.expectAssertions = function(min, max) {
+    if (typeof(max) == "undefined") {
+        max = min;
+    }
+    if (typeof(min) != "number" || typeof(max) != "number" ||
+        min < 0 || max < min) {
+        throw "bad parameter to expectAssertions";
+    }
+    TestRunner._expectedMinAsserts = min;
+    TestRunner._expectedMaxAsserts = max;
+}
+
 /**
  * This function is called after generating the summary.
 **/
 TestRunner.onComplete = null;
 
 /**
  * Adds a failed test case to a list so we can rerun only the failed tests
  **/
@@ -333,16 +349,18 @@ TestRunner.runNextTest = function() {
     {
         var url = TestRunner._urls[TestRunner._currentTest];
         TestRunner.currentTestURL = url;
 
         $("current-test-path").innerHTML = url;
 
         TestRunner._currentTestStartTime = new Date().valueOf();
         TestRunner._timeoutFactor = 1;
+        TestRunner._expectedMinAsserts = 0;
+        TestRunner._expectedMaxAsserts = 0;
 
         TestRunner.log("TEST-START | " + url); // used by automation.py
 
         TestRunner._makeIframe(url, 0);
     } else {
         $("current-test").innerHTML = "<b>Finished</b>";
         TestRunner._makeIframe("about:blank", 0);
 
@@ -396,16 +414,26 @@ TestRunner.runNextTest = function() {
 TestRunner.expectChildProcessCrash = function() {
     TestRunner._expectingProcessCrash = true;
 };
 
 /**
  * This stub is called by SimpleTest when a test is finished.
 **/
 TestRunner.testFinished = function(tests) {
+    // Prevent a test from calling finish() multiple times before we
+    // have a chance to unload it.
+    if (TestRunner._currentTest == TestRunner._lastTestFinished) {
+        TestRunner.error("TEST-UNEXPECTED-FAIL | " +
+                         TestRunner.currentTestURL +
+                         " | called finish() multiple times");
+        return;
+    }
+    TestRunner._lastTestFinished = TestRunner._currentTest;
+
     function cleanUpCrashDumpFiles() {
         if (!SpecialPowers.removeExpectedCrashDumpFiles(TestRunner._expectingProcessCrash)) {
             TestRunner.error("TEST-UNEXPECTED-FAIL | " +
                              TestRunner.currentTestURL +
                              " | This test did not leave any crash dumps behind, but we were expecting some!");
             tests.push({ result: false });
         }
         var unexpectedCrashDumpFiles =
@@ -435,30 +463,61 @@ TestRunner.testFinished = function(tests
         }
 
         var runtime = new Date().valueOf() - TestRunner._currentTestStartTime;
         TestRunner.log("TEST-END | " +
                        TestRunner.currentTestURL +
                        " | finished in " + runtime + "ms");
 
         TestRunner.updateUI(tests);
-        TestRunner._currentTest++;
-        if (TestRunner.runSlower) {
-            setTimeout(TestRunner.runNextTest, 1000);
+
+        var interstitialURL;
+        if ($('testframe').contentWindow.location.protocol == "chrome:") {
+            interstitialURL = "tests/SimpleTest/iframe-between-tests.html";
         } else {
-            TestRunner.runNextTest();
+            interstitialURL = "/tests/SimpleTest/iframe-between-tests.html";
         }
+        TestRunner._makeIframe(interstitialURL, 0);
     }
 
     SpecialPowers.executeAfterFlushingMessageQueue(function() {
         cleanUpCrashDumpFiles();
         SpecialPowers.flushPrefEnv(runNextTest);
     });
 };
 
+TestRunner.testUnloaded = function() {
+    if (SpecialPowers.isDebugBuild) {
+        var newAssertionCount = SpecialPowers.assertionCount();
+        var numAsserts = newAssertionCount - TestRunner._lastAssertionCount;
+        TestRunner._lastAssertionCount = newAssertionCount;
+
+        var url = TestRunner._urls[TestRunner._currentTest];
+        var max = TestRunner._expectedMaxAsserts;
+        var min = TestRunner._expectedMinAsserts;
+        if (numAsserts > max) {
+            // WHEN ENABLING, change "log" to "error" and "DETCEPXENU"
+            // to "UNEXPECTED".
+            TestRunner.log("TEST-DETCEPXENU-FAIL | " + url + " | Assertion count " + numAsserts + " is greater than expected range " + min + "-" + max + " assertions.");
+        } else if (numAsserts < min) {
+            // WHEN ENABLING, change "log" to "error" and "DETCEPXENU"
+            // to "UNEXPECTED".
+            TestRunner.log("TEST-DETCEPXENU-PASS | " + url + " | Assertion count " + numAsserts + " is less than expected range " + min + "-" + max + " assertions.");
+        } else if (numAsserts > 0) {
+            TestRunner.log("TEST-KNOWN-FAIL | " + url + " | Assertion count " + numAsserts + " within expected range " + min + "-" + max + " assertions.");
+        }
+    }
+    TestRunner._currentTest++;
+    if (TestRunner.runSlower) {
+        setTimeout(TestRunner.runNextTest, 1000);
+    } else {
+        TestRunner.runNextTest();
+    }
+};
+
 /**
  * Get the results.
  */
 TestRunner.countResults = function(tests) {
   var nOK = 0;
   var nNotOK = 0;
   var nTodo = 0;
   for (var i = 0; i < tests.length; ++i) {
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/SimpleTest/iframe-between-tests.html
@@ -0,0 +1,12 @@
+<title>iframe for between tests</title>
+<!--
+  This page exists so that our accounting for assertions correctly
+  counts assertions that happen while leaving a page.  We load this page
+  after a test finishes, check the assertion counts, and then go on to
+  load the next.
+-->
+<script>
+window.addEventListener("load", function() {
+  (parent.TestRunner || parent.wrappedJSObject.TestRunner).testUnloaded();
+});
+</script>
--- a/testing/specialpowers/content/specialpowersAPI.js
+++ b/testing/specialpowers/content/specialpowersAPI.js
@@ -1201,16 +1201,20 @@ SpecialPowersAPI.prototype = {
     return el.dispatchEvent(event);
   },
 
   get isDebugBuild() {
     delete this.isDebugBuild;
     var debug = Cc["@mozilla.org/xpcom/debug;1"].getService(Ci.nsIDebug2);
     return this.isDebugBuild = debug.isDebugBuild;
   },
+  assertionCount: function() {
+    var debugsvc = Cc['@mozilla.org/xpcom/debug;1'].getService(Ci.nsIDebug2);
+    return debugsvc.assertionCount;
+  },
 
   /**
    * Get the message manager associated with an <iframe mozbrowser>.
    */
   getBrowserFrameMessageManager: function(aFrameElement) {
     return this.wrap(aFrameElement.QueryInterface(Ci.nsIFrameLoaderOwner)
                                   .frameLoader
                                   .messageManager);