Bug 847275 patch 1: Make assertions cause test failures in browser-chrome mochitest. r=dao
authorL. David Baron <dbaron@dbaron.org>
Mon, 09 Sep 2013 12:21:24 +0200
changeset 146170 12d4dea8cdfb2cd3fbeec30cdd2775900ef168e5
parent 146169 4f9563af1fa1274bbfde1466a44a9253e1a2a32c
child 146171 b560ee360b68cc543a8aa38e721bbc9a4ea3c9ea
push id25244
push userryanvm@gmail.com
push dateMon, 09 Sep 2013 20:03:14 +0000
treeherdermozilla-central@f320b8c034bd [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdao
bugs847275
milestone26.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 847275 patch 1: Make assertions cause test failures in browser-chrome mochitest. r=dao I tested all three error messages manually by making different SimpleTest.expectAssertions() calls in browser/base/content/test/browser_aboutHome.js .
testing/mochitest/browser-test.js
testing/mochitest/tests/SimpleTest/TestRunner.js
--- a/testing/mochitest/browser-test.js
+++ b/testing/mochitest/browser-test.js
@@ -83,16 +83,17 @@ Tester.prototype = {
   Promise: null,
 
   repeat: 0,
   runUntilFailure: false,
   checker: null,
   currentTestIndex: -1,
   lastStartTime: null,
   openedWindows: null,
+  lastAssertionCount: 0,
 
   get currentTest() {
     return this.tests[this.currentTestIndex];
   },
   get done() {
     return this.currentTestIndex == this.tests.length - 1;
   },
 
@@ -310,16 +311,48 @@ Tester.prototype = {
       // Notify a long running test problem if it didn't end up in a timeout.
       if (this.currentTest.unexpectedTimeouts && !this.currentTest.timedOut) {
         let msg = "This test exceeded the timeout threshold. It should be " +
                   "rewritten or split up. If that's not possible, use " +
                   "requestLongerTimeout(N), but only as a last resort.";
         this.currentTest.addResult(new testResult(false, msg, "", false));
       }
 
+      // If we're in a debug build, check assertion counts.  This code
+      // is similar to the code in TestRunner.testUnloaded in
+      // TestRunner.js used for all other types of mochitests.
+      let debugsvc = Cc["@mozilla.org/xpcom/debug;1"].getService(Ci.nsIDebug2);
+      if (debugsvc.isDebugBuild) {
+        let newAssertionCount = debugsvc.assertionCount;
+        let numAsserts = newAssertionCount - this.lastAssertionCount;
+        this.lastAssertionCount = newAssertionCount;
+
+        let max = testScope.__expectedMaxAsserts;
+        let min = testScope.__expectedMinAsserts;
+        if (numAsserts > max) {
+          let msg = "Assertion count " + numAsserts +
+                    " is greater than expected range " +
+                    min + "-" + max + " assertions.";
+          // TEST-UNEXPECTED-FAIL
+          this.currentTest.addResult(new testResult(false, msg, "", false));
+        } else if (numAsserts < min) {
+          let msg = "Assertion count " + numAsserts +
+                    " is less than expected range " +
+                    min + "-" + max + " assertions.";
+          // TEST-UNEXPECTED-PASS
+          this.currentTest.addResult(new testResult(false, msg, "", true));
+        } else if (numAsserts > 0) {
+          let msg = "Assertion count " + numAsserts +
+                    " is within expected range " +
+                    min + "-" + max + " assertions.";
+          // TEST-KNOWN-FAIL
+          this.currentTest.addResult(new testResult(true, msg, "", true));
+        }
+      }
+
       // Note the test run time
       let time = Date.now() - this.lastStartTime;
       this.dumper.dump("INFO TEST-END | " + this.currentTest.path + " | finished in " + time + "ms\n");
       this.currentTest.setDuration(time);
 
       testScope.destroy();
       this.currentTest.scope = null;
     }
@@ -401,17 +434,17 @@ Tester.prototype = {
     // Import utils in the test scope.
     this.currentTest.scope.EventUtils = this.EventUtils;
     this.currentTest.scope.SimpleTest = this.SimpleTest;
     this.currentTest.scope.gTestPath = this.currentTest.path;
     this.currentTest.scope.Task = this.Task;
     this.currentTest.scope.Promise = this.Promise;
 
     // Override SimpleTest methods with ours.
-    ["ok", "is", "isnot", "ise", "todo", "todo_is", "todo_isnot", "info"].forEach(function(m) {
+    ["ok", "is", "isnot", "ise", "todo", "todo_is", "todo_isnot", "info", "expectAssertions"].forEach(function(m) {
       this.SimpleTest[m] = this[m];
     }, this.currentTest.scope);
 
     //load the tools to work with chrome .jar and remote
     try {
       this._scriptLoader.loadSubScript("chrome://mochikit/content/chrome-harness.js", this.currentTest.scope);
     } catch (ex) { /* no chrome-harness tools */ }
 
@@ -678,16 +711,30 @@ function testScope(aTester, aTest) {
   this.expectUncaughtException = function test_expectUncaughtException(aExpecting) {
     self.SimpleTest.expectUncaughtException(aExpecting);
   };
 
   this.ignoreAllUncaughtExceptions = function test_ignoreAllUncaughtExceptions(aIgnoring) {
     self.SimpleTest.ignoreAllUncaughtExceptions(aIgnoring);
   };
 
+  this.expectAssertions = function test_expectAssertions(aMin, aMax) {
+    let min = aMin;
+    let max = aMax;
+    if (typeof(max) == "undefined") {
+      max = min;
+    }
+    if (typeof(min) != "number" || typeof(max) != "number" ||
+        min < 0 || max < min) {
+      throw "bad parameter to expectAssertions";
+    }
+    self.__expectedMinAsserts = min;
+    self.__expectedMaxAsserts = max;
+  };
+
   this.finish = function test_finish() {
     self.__done = true;
     if (self.__waitTimer) {
       self.executeSoon(function() {
         if (self.__done && self.__waitTimer) {
           clearTimeout(self.__waitTimer);
           self.__waitTimer = null;
           self.__tester.nextTest();
@@ -698,16 +745,18 @@ function testScope(aTester, aTest) {
 }
 testScope.prototype = {
   __done: true,
   __generator: null,
   __tasks: null,
   __waitTimer: null,
   __cleanupFunctions: [],
   __timeoutFactor: 1,
+  __expectedMinAsserts: 0,
+  __expectedMaxAsserts: 0,
 
   EventUtils: {},
   SimpleTest: {},
   Task: null,
   Promise: null,
 
   /**
    * Add a test function which is a Task function.
--- a/testing/mochitest/tests/SimpleTest/TestRunner.js
+++ b/testing/mochitest/tests/SimpleTest/TestRunner.js
@@ -457,16 +457,19 @@ TestRunner.testFinished = function(tests
 
     SpecialPowers.executeAfterFlushingMessageQueue(function() {
         cleanUpCrashDumpFiles();
         SpecialPowers.flushPermissions(function () { SpecialPowers.flushPrefEnv(runNextTest); });
     });
 };
 
 TestRunner.testUnloaded = function() {
+    // If we're in a debug build, check assertion counts.  This code is
+    // similar to the code in Tester_nextTest in browser-test.js used
+    // for browser-chrome mochitests.
     if (SpecialPowers.isDebugBuild) {
         var newAssertionCount = SpecialPowers.assertionCount();
         var numAsserts = newAssertionCount - TestRunner._lastAssertionCount;
         TestRunner._lastAssertionCount = newAssertionCount;
 
         var url = TestRunner._urls[TestRunner._currentTest];
         var max = TestRunner._expectedMaxAsserts;
         var min = TestRunner._expectedMinAsserts;