Bug 1365649 - Improve browser-chrome self-tests and the "fail-if" manifest property handling. r=Mossop
☠☠ backed out by 398d167b26f9 ☠ ☠
authorPaolo Amadini <paolo.mozmail@amadzone.org>
Sat, 27 May 2017 09:12:02 +0100
changeset 585744 6da58c7bb247d3e879012bea8d848eb68f16e36e
parent 585722 6a235092bda7c20cb83651d3404284fe36d04c1b
child 585745 b59573695517b2a464f22e00bd3792c50dcf371a
push id61188
push usermaglione.k@gmail.com
push dateSun, 28 May 2017 22:11:50 +0000
reviewersMossop
bugs1365649
milestone55.0a1
Bug 1365649 - Improve browser-chrome self-tests and the "fail-if" manifest property handling. r=Mossop The browser-chrome self-test files now use the setExpectedFailuresForSelfTest function to specify the exact number of assertion failures that will be triggered. Also, most failures are now intercepted when specifying the "fail-if" property in a "browser.ini" manifest, while previously only those triggered using the "ok" function were intercepted. This allows re-enabling several browser-chome self-tests. MozReview-Commit-ID: DlDjWaJPfvH
testing/mochitest/browser-harness.xul
testing/mochitest/browser-test.js
testing/mochitest/tests/browser/browser.ini
testing/mochitest/tests/browser/browser_async.js
testing/mochitest/tests/browser/browser_fail.js
testing/mochitest/tests/browser/browser_fail_add_task.js
testing/mochitest/tests/browser/browser_fail_async.js
testing/mochitest/tests/browser/browser_fail_async_throw.js
testing/mochitest/tests/browser/browser_fail_fp.js
testing/mochitest/tests/browser/browser_fail_if.js
testing/mochitest/tests/browser/browser_fail_pf.js
testing/mochitest/tests/browser/browser_fail_throw.js
testing/mochitest/tests/browser/browser_fail_uncaught_rejection.js
--- a/testing/mochitest/browser-harness.xul
+++ b/testing/mochitest/browser-harness.xul
@@ -127,16 +127,19 @@
         return this.results.filter(t => !t.info && !t.todo && t.pass).length;
       },
       get todoCount() {
         return this.results.filter(t => !t.info && t.todo && t.pass).length;
       },
       get failCount() {
         return this.results.filter(t => !t.info && !t.pass).length;
       },
+      get allowedFailureCount() {
+        return this.results.filter(t => t.allowedFailure).length;
+      },
 
       addResult: function addResult(result) {
         this.lastOutputTime = Date.now();
         this.results.push(result);
 
         if (result.info) {
           if (result.msg) {
             this.dumper.structuredLogger.info(result.msg);
--- a/testing/mochitest/browser-test.js
+++ b/testing/mochitest/browser-test.js
@@ -160,44 +160,41 @@ function Tester(aTests, structuredLogger
   SIMPLETEST_OVERRIDES.forEach(m => {
     this.SimpleTestOriginal[m] = this.SimpleTest[m];
   });
 
   this._coverageCollector = null;
 
   this._toleratedUncaughtRejections = null;
   this._uncaughtErrorObserver = ({message, date, fileName, stack, lineNumber}) => {
-    let error = message;
+    let ex = message;
     if (fileName || lineNumber) {
-      error = {
+      ex = {
         fileName: fileName,
         lineNumber: lineNumber,
         message: message,
         toString: function() {
           return message;
         }
       };
     }
 
     // We may have a whitelist of rejections we wish to tolerate.
-    let tolerate = this._toleratedUncaughtRejections &&
+    let pass = this._toleratedUncaughtRejections &&
       this._toleratedUncaughtRejections.indexOf(message) != -1;
     let name = "A promise chain failed to handle a rejection: ";
-    if (tolerate) {
+    if (pass) {
       name = "WARNING: (PLEASE FIX THIS AS PART OF BUG 1077403) " + name;
     }
 
-    this.currentTest.addResult(
-      new testResult(
-	      /*success*/tolerate,
-        /*name*/name,
-        /*error*/error,
-        /*known*/tolerate,
-        /*stack*/stack));
-    };
+    this.currentTest.addResult(new testResult({
+      pass, name, ex, todo: pass, stack,
+      allowFailure: this.currentTest.allowFailure,
+    }));
+  };
 }
 Tester.prototype = {
   EventUtils: {},
   SimpleTest: {},
   Task: null,
   ContentTask: null,
   ExtensionTestUtils: null,
   Assert: null,
@@ -276,19 +273,21 @@ Tester.prototype = {
     let baseMsg = timedOut ? "Found a {elt} after previous test timed out"
                            : this.currentTest ? "Found an unexpected {elt} at the end of test run"
                                               : "Found an unexpected {elt}";
 
     // Remove stale tabs
     if (this.currentTest && window.gBrowser && gBrowser.tabs.length > 1) {
       while (gBrowser.tabs.length > 1) {
         let lastTab = gBrowser.tabContainer.lastChild;
-        let msg = baseMsg.replace("{elt}", "tab") +
-                  ": " + lastTab.linkedBrowser.currentURI.spec;
-        this.currentTest.addResult(new testResult(false, msg, "", false));
+        this.currentTest.addResult(new testResult({
+          name: baseMsg.replace("{elt}", "tab") + ": " +
+                lastTab.linkedBrowser.currentURI.spec,
+          allowFailure: this.currentTest.allowFailure,
+        }));
         gBrowser.removeTab(lastTab);
       }
     }
 
     // Replace the last tab with a fresh one
     if (window.gBrowser) {
       let newTab = gBrowser.addTab("about:blank", { skipAnimation: true });
       gBrowser.removeTab(gBrowser.selectedTab, { skipPermitUnload: true });
@@ -310,17 +309,20 @@ Tester.prototype = {
           break;
         case null:
           type = "unknown window with document URI: " + win.document.documentURI +
                  " and title: " + win.document.title;
           break;
         }
         let msg = baseMsg.replace("{elt}", type);
         if (this.currentTest) {
-          this.currentTest.addResult(new testResult(false, msg, "", false));
+          this.currentTest.addResult(new testResult({
+            name: msg,
+            allowFailure: this.currentTest.allowFailure,
+          }));
         } else {
           if (!createdFakeTestForLogging) {
             createdFakeTestForLogging = true;
             this.structuredLogger.testStart("browser-test.js");
           }
           this.failuresFromInitialWindowState++;
           this.structuredLogger.testStatus("browser-test.js",
                                            msg, "FAIL", false, "");
@@ -429,55 +431,69 @@ Tester.prototype = {
       // next one.
       let testScope = this.currentTest.scope;
       while (testScope.__cleanupFunctions.length > 0) {
         let func = testScope.__cleanupFunctions.shift();
         try {
           yield func.apply(testScope);
         }
         catch (ex) {
-          this.currentTest.addResult(new testResult(false, "Cleanup function threw an exception", ex, false));
+          this.currentTest.addResult(new testResult({
+            name: "Cleanup function threw an exception",
+            ex,
+            allowFailure: this.currentTest.allowFailure,
+          }));
         }
       }
 
       if (this.currentTest.passCount === 0 &&
           this.currentTest.failCount === 0 &&
           this.currentTest.todoCount === 0) {
-        this.currentTest.addResult(new testResult(false, "This test contains no passes, no fails and no todos. Maybe it threw a silent exception? Make sure you use waitForExplicitFinish() if you need it.", "", false));
-      }
-
-      if (testScope.__expected == 'fail' && testScope.__num_failed <= 0) {
-        this.currentTest.addResult(new testResult(false, "We expected at least one assertion to fail because this test file was marked as fail-if in the manifest!", "", true));
+        this.currentTest.addResult(new testResult({
+          name: "This test contains no passes, no fails and no todos. Maybe" +
+                " it threw a silent exception? Make sure you use" +
+                " waitForExplicitFinish() if you need it.",
+        }));
       }
 
       this.Promise.Debugging.flushUncaughtErrors();
 
       let winUtils = window.QueryInterface(Ci.nsIInterfaceRequestor)
                            .getInterface(Ci.nsIDOMWindowUtils);
       if (winUtils.isTestControllingRefreshes) {
-        this.currentTest.addResult(new testResult(false, "test left refresh driver under test control", "", false));
+        this.currentTest.addResult(new testResult({
+          name: "test left refresh driver under test control",
+        }));
         winUtils.restoreNormalRefresh();
       }
 
       if (this.SimpleTest.isExpectingUncaughtException()) {
-        this.currentTest.addResult(new testResult(false, "expectUncaughtException was called but no uncaught exception was detected!", "", false));
+        this.currentTest.addResult(new testResult({
+          name: "expectUncaughtException was called but no uncaught" +
+                " exception was detected!",
+          allowFailure: this.currentTest.allowFailure,
+        }));
       }
 
       Object.keys(window).forEach(function (prop) {
         if (parseInt(prop) == prop) {
           // This is a string which when parsed as an integer and then
           // stringified gives the original string.  As in, this is in fact a
           // string representation of an integer, so an index into
           // window.frames.  Skip those.
           return;
         }
         if (this._globalProperties.indexOf(prop) == -1) {
           this._globalProperties.push(prop);
-          if (this._globalPropertyWhitelist.indexOf(prop) == -1)
-            this.currentTest.addResult(new testResult(false, "test left unexpected property on window: " + prop, "", false));
+          if (this._globalPropertyWhitelist.indexOf(prop) == -1) {
+            this.currentTest.addResult(new testResult({
+              name: "test left unexpected property on window: " + prop,
+              allowFailure: this.currentTest.allowFailure,
+            }));
+          }
         }
       }, this);
 
       // Clear document.popupNode.  The test could have set it to a custom value
       // for its own purposes, nulling it out it will go back to the default
       // behavior of returning the last opened popup.
       document.popupNode = null;
 
@@ -497,52 +513,82 @@ Tester.prototype = {
               this.structuredLogger.info(msg);
             }
           }
         }
       }
 
       // Notify a long running test problem if it didn't end up in a timeout.
       if (this.currentTest.unexpectedTimeouts && !this.currentTest.timedOut) {
-        let msg = "This test exceeded the timeout threshold. It should be " +
-                  "rewritten or split up. If that's not possible, use " +
-                  "requestLongerTimeout(N), but only as a last resort.";
-        this.currentTest.addResult(new testResult(false, msg, "", false));
+        this.currentTest.addResult(new testResult({
+          name: "This test exceeded the timeout threshold. It should be" +
+                " rewritten or split up. If that's not possible, use" +
+                " requestLongerTimeout(N), but only as a last resort.",
+        }));
       }
 
       // If we're in a debug build, check assertion counts.  This code
       // is similar to the code in TestRunner.testUnloaded in
       // TestRunner.js used for all other types of mochitests.
       let debugsvc = Cc["@mozilla.org/xpcom/debug;1"].getService(Ci.nsIDebug2);
       if (debugsvc.isDebugBuild) {
         let newAssertionCount = debugsvc.assertionCount;
         let numAsserts = newAssertionCount - this.lastAssertionCount;
         this.lastAssertionCount = newAssertionCount;
 
         let max = testScope.__expectedMaxAsserts;
         let min = testScope.__expectedMinAsserts;
         if (numAsserts > max) {
-          let msg = "Assertion count " + numAsserts +
-                    " is greater than expected range " +
-                    min + "-" + max + " assertions.";
-          // TEST-UNEXPECTED-FAIL (TEMPORARILY TEST-KNOWN-FAIL)
-          //this.currentTest.addResult(new testResult(false, msg, "", false));
-          this.currentTest.addResult(new testResult(true, msg, "", true));
+          // TEST-UNEXPECTED-FAIL
+          this.currentTest.addResult(new testResult({
+            name: "Assertion count " + numAsserts +
+                  " is greater than expected range " +
+                  min + "-" + max + " assertions.",
+            pass: true, // TEMPORARILY TEST-KNOWN-FAIL
+            todo: true,
+            allowFailure: this.currentTest.allowFailure,
+          }));
         } else if (numAsserts < min) {
-          let msg = "Assertion count " + numAsserts +
-                    " is less than expected range " +
-                    min + "-" + max + " assertions.";
           // TEST-UNEXPECTED-PASS
-          this.currentTest.addResult(new testResult(false, msg, "", true));
+          this.currentTest.addResult(new testResult({
+            name: "Assertion count " + numAsserts +
+                  " is less than expected range " +
+                  min + "-" + max + " assertions.",
+            todo: true,
+            allowFailure: this.currentTest.allowFailure,
+          }));
         } else if (numAsserts > 0) {
-          let msg = "Assertion count " + numAsserts +
-                    " is within expected range " +
-                    min + "-" + max + " assertions.";
           // TEST-KNOWN-FAIL
-          this.currentTest.addResult(new testResult(true, msg, "", true));
+          this.currentTest.addResult(new testResult({
+            name: "Assertion count " + numAsserts +
+                  " is within expected range " +
+                  min + "-" + max + " assertions.",
+            pass: true,
+            todo: true,
+            allowFailure: this.currentTest.allowFailure,
+          }));
+        }
+      }
+
+      if (this.currentTest.allowFailure) {
+        if (this.currentTest.expectedAllowedFailureCount) {
+          this.currentTest.addResult(new testResult({
+            name: "Expected " +
+                  this.currentTest.expectedAllowedFailureCount +
+                  " failures in this file, got " +
+                  this.currentTest.allowedFailureCount + ".",
+            pass: this.currentTest.expectedAllowedFailureCount ==
+                  this.currentTest.allowedFailureCount,
+          }));
+        } else if (this.currentTest.allowedFailureCount == 0) {
+          this.currentTest.addResult(new testResult({
+            name: "We expect at least one assertion to fail because this" +
+                  " test file is marked as fail-if in the manifest.",
+            todo: true,
+          }));
         }
       }
 
       // Dump memory stats for main thread.
       if (Cc["@mozilla.org/xre/runtime;1"]
           .getService(Ci.nsIXULRuntime)
           .processType == Ci.nsIXULRuntime.PROCESS_TYPE_DEFAULT)
       {
@@ -684,23 +730,27 @@ Tester.prototype = {
     this.currentTest.scope.gTestPath = this.currentTest.path;
     this.currentTest.scope.Task = this.Task;
     this.currentTest.scope.ContentTask = this.ContentTask;
     this.currentTest.scope.BrowserTestUtils = this.BrowserTestUtils;
     this.currentTest.scope.TestUtils = this.TestUtils;
     this.currentTest.scope.ExtensionTestUtils = this.ExtensionTestUtils;
     // Pass a custom report function for mochitest style reporting.
     this.currentTest.scope.Assert = new this.Assert(function(err, message, stack) {
-      let res;
-      if (err) {
-        res = new testResult(false, err.message, err.stack, false, err.stack);
-      } else {
-        res = new testResult(true, message, "", false, stack);
-      }
-      currentTest.addResult(res);
+      currentTest.addResult(new testResult(err ? {
+        name: err.message,
+        ex: err.stack,
+        stack: err.stack,
+        allowFailure: currentTest.allowFailure,
+      } : {
+        name: message,
+        pass: true,
+        stack,
+        allowFailure: currentTest.allowFailure,
+      }));
     });
 
     this.ContentTask.setTestScope(currentScope);
 
     // Allow Assert.jsm methods to be tacked to the current scope.
     this.currentTest.scope.export_assertions = function() {
       for (let func in this.Assert) {
         this[func] = this.Assert[func].bind(this.Assert);
@@ -723,17 +773,20 @@ Tester.prototype = {
     var headPath = currentTestDirPath + "/head.js";
     try {
       this._scriptLoader.loadSubScript(headPath, this.currentTest.scope);
     } catch (ex) {
       // Ignore if no head.js exists, but report all other errors.  Note this
       // will also ignore an existing head.js attempting to import a missing
       // module - see bug 755558 for why this strategy is preferred anyway.
       if (!/^Error opening input stream/.test(ex.toString())) {
-       this.currentTest.addResult(new testResult(false, "head.js import threw an exception", ex, false));
+       this.currentTest.addResult(new testResult({
+         name: "head.js import threw an exception",
+         ex,
+       }));
       }
     }
 
     // Import the test script.
     try {
       this._scriptLoader.loadSubScript(this.currentTest.path,
                                        this.currentTest.scope);
       this.Promise.Debugging.flushUncaughtErrors();
@@ -747,36 +800,42 @@ Tester.prototype = {
         let Promise = this.Promise;
         this.Task.spawn(function*() {
           let task;
           while ((task = this.__tasks.shift())) {
             this.SimpleTest.info("Entering test " + task.name);
             try {
               yield task();
             } catch (ex) {
-              let isExpected = !!this.SimpleTest.isExpectingUncaughtException();
-              let stack = (typeof ex == "object" && "stack" in ex)?ex.stack:null;
-              let name = "Uncaught exception";
-              let result = new testResult(isExpected, name, ex, false, stack);
-              currentTest.addResult(result);
+              currentTest.addResult(new testResult({
+                name: "Uncaught exception",
+                pass: this.SimpleTest.isExpectingUncaughtException(),
+                ex,
+                stack: (typeof ex == "object" && "stack" in ex) ? ex.stack : null,
+                allowFailure: currentTest.allowFailure,
+              }));
             }
             Promise.Debugging.flushUncaughtErrors();
             this.SimpleTest.info("Leaving test " + task.name);
           }
           this.finish();
         }.bind(currentScope));
       } else if (typeof this.currentTest.scope.test == "function") {
         this.currentTest.scope.test();
       } else {
         throw "This test didn't call add_task, nor did it define a generatorTest() function, nor did it define a test() function, so we don't know how to run it.";
       }
     } catch (ex) {
-      let isExpected = !!this.SimpleTest.isExpectingUncaughtException();
       if (!this.SimpleTest.isIgnoringAllUncaughtExceptions()) {
-        this.currentTest.addResult(new testResult(isExpected, "Exception thrown", ex, false));
+        this.currentTest.addResult(new testResult({
+          name: "Exception thrown",
+          pass: this.SimpleTest.isExpectingUncaughtException(),
+          ex,
+          allowFailure: this.currentTest.allowFailure,
+        }));
         this.SimpleTest.expectUncaughtException(false);
       } else {
         this.currentTest.addResult(new testMessage("Exception thrown: " + ex));
       }
       this.currentTest.scope.finish();
     }
 
     // If the test ran synchronously, move to the next test, otherwise the test
@@ -819,123 +878,141 @@ Tester.prototype = {
         const MAX_UNEXPECTED_TIMEOUTS = 10;
         if (Date.now() - self.currentTest.lastOutputTime < (gTimeoutSeconds / 2) * 1000 &&
             ++self.currentTest.unexpectedTimeouts <= MAX_UNEXPECTED_TIMEOUTS) {
             self.currentTest.scope.__waitTimer =
               setTimeout(timeoutFn, gTimeoutSeconds * 1000);
           return;
         }
 
-        self.currentTest.addResult(new testResult(false, "Test timed out", null, false));
+        self.currentTest.addResult(new testResult({ name: "Test timed out" }));
         self.currentTest.timedOut = true;
         self.currentTest.scope.__waitTimer = null;
         self.nextTest();
       }, gTimeoutSeconds * 1000]);
     }
   },
 
   QueryInterface: function(aIID) {
     if (aIID.equals(Ci.nsIConsoleListener) ||
         aIID.equals(Ci.nsISupports))
       return this;
 
     throw Components.results.NS_ERROR_NO_INTERFACE;
   }
 };
 
-function testResult(aCondition, aName, aDiag, aIsTodo, aStack) {
-  this.name = aName;
+/**
+ * Represents the result of one test assertion. This is described with a string
+ * in traditional logging, and has a "status" and "expected" property used in
+ * structured logging. Normally, results are mapped as follows:
+ *
+ *   pass:    todo:    Added to:    Described as:           Status:  Expected:
+ *     true     false    passCount    TEST-PASS               PASS     PASS
+ *     true     true     todoCount    TEST-KNOWN-FAIL         FAIL     FAIL
+ *     false    false    failCount    TEST-UNEXPECTED-FAIL    FAIL     PASS
+ *     false    true     failCount    TEST-UNEXPECTED-PASS    PASS     FAIL
+ *
+ * The "allowFailure" argument indicates that this is one of the assertions that
+ * should be allowed to fail, for example because "fail-if" is true for the
+ * current test file in the manifest. In this case, results are mapped this way:
+ *
+ *   pass:    todo:    Added to:    Described as:           Status:  Expected:
+ *     true     false    passCount    TEST-PASS               PASS     PASS
+ *     true     true     todoCount    TEST-KNOWN-FAIL         FAIL     FAIL
+ *     false    false    todoCount    TEST-KNOWN-FAIL         FAIL     FAIL
+ *     false    true     todoCount    TEST-KNOWN-FAIL         FAIL     FAIL
+ */
+function testResult({ name, pass, todo, ex, stack, allowFailure }) {
+  this.info = false;
+  this.name = name;
   this.msg = "";
 
-  this.info = false;
-  this.pass = !!aCondition;
-  this.todo = aIsTodo;
+  if (allowFailure && !pass) {
+    this.allowedFailure = true;
+    this.pass = true;
+    this.todo = true;
+  } else {
+    this.pass = !!pass;
+    this.todo = todo;
+  }
+
+  this.expected = this.todo ? "FAIL" : "PASS";
 
   if (this.pass) {
-    if (aIsTodo) {
-      this.status = "FAIL";
-      this.expected = "FAIL";
-    } else {
-      this.status = "PASS";
-      this.expected = "PASS";
-    }
+    this.status = this.expected;
+    return;
+  }
 
-  } else {
-    if (aDiag) {
-      if (typeof aDiag == "object" && "fileName" in aDiag) {
-        // we have an exception - print filename and linenumber information
-        this.msg += "at " + aDiag.fileName + ":" + aDiag.lineNumber + " - ";
-      }
-      this.msg += String(aDiag);
+  this.status = this.todo ? "PASS" : "FAIL";
+
+  if (ex) {
+    if (typeof ex == "object" && "fileName" in ex) {
+      // we have an exception - print filename and linenumber information
+      this.msg += "at " + ex.fileName + ":" + ex.lineNumber + " - ";
     }
-    if (aStack) {
-      this.msg += "\nStack trace:\n";
-      let normalized;
-      if (aStack instanceof Components.interfaces.nsIStackFrame) {
-        let frames = [];
-        for (let frame = aStack; frame; frame = frame.caller) {
-          frames.push(frame.filename + ":" + frame.name + ":" + frame.lineNumber);
-        }
-        normalized = frames.join("\n");
-      } else {
-        normalized = "" + aStack;
+    this.msg += String(ex);
+  }
+
+  if (stack) {
+    this.msg += "\nStack trace:\n";
+    let normalized;
+    if (stack instanceof Components.interfaces.nsIStackFrame) {
+      let frames = [];
+      for (let frame = stack; frame; frame = frame.caller) {
+        frames.push(frame.filename + ":" + frame.name + ":" + frame.lineNumber);
       }
-      this.msg += Task.Debugging.generateReadableStack(normalized, "    ");
+      normalized = frames.join("\n");
+    } else {
+      normalized = "" + stack;
     }
-    if (aIsTodo) {
-      this.status = "PASS";
-      this.expected = "FAIL";
-    } else {
-      this.status = "FAIL";
-      this.expected = "PASS";
-    }
+    this.msg += Task.Debugging.generateReadableStack(normalized, "    ");
+  }
 
-    if (gConfig.debugOnFailure) {
-      // You've hit this line because you requested to break into the
-      // debugger upon a testcase failure on your test run.
-      debugger;
-    }
+  if (gConfig.debugOnFailure) {
+    // You've hit this line because you requested to break into the
+    // debugger upon a testcase failure on your test run.
+    debugger;
   }
 }
 
-function testMessage(aName) {
-  this.msg = aName || "";
+function testMessage(msg) {
+  this.msg = msg || "";
   this.info = true;
 }
 
 // Need to be careful adding properties to this object, since its properties
 // cannot conflict with global variables used in tests.
 function testScope(aTester, aTest, expected) {
   this.__tester = aTester;
-  this.__expected = expected;
-  this.__num_failed = 0;
+
+  aTest.allowFailure = expected == "fail";
 
   var self = this;
-  this.ok = function test_ok(condition, name, diag, stack) {
-    if (self.__expected == 'fail') {
-        if (!condition) {
-          self.__num_failed++;
-          condition = true;
-        }
-    }
-
-    aTest.addResult(new testResult(condition, name, diag, false,
-                                   stack ? stack : Components.stack.caller));
+  this.ok = function test_ok(condition, name, ex, stack) {
+    aTest.addResult(new testResult({
+      name, pass: condition, ex,
+      stack: stack || Components.stack.caller,
+      allowFailure: aTest.allowFailure,
+    }));
   };
   this.is = function test_is(a, b, name) {
     self.ok(a == b, name, "Got " + a + ", expected " + b, false,
             Components.stack.caller);
   };
   this.isnot = function test_isnot(a, b, name) {
     self.ok(a != b, name, "Didn't expect " + a + ", but got it", false,
             Components.stack.caller);
   };
-  this.todo = function test_todo(condition, name, diag, stack) {
-    aTest.addResult(new testResult(!condition, name, diag, true,
-                                   stack ? stack : Components.stack.caller));
+  this.todo = function test_todo(condition, name, ex, stack) {
+    aTest.addResult(new testResult({
+      name, pass: !condition, todo: true, ex,
+      stack: stack || Components.stack.caller,
+      allowFailure: aTest.allowFailure,
+    }));
   };
   this.todo_is = function test_todo_is(a, b, name) {
     self.todo(a == b, name, "Got " + a + ", expected " + b,
               Components.stack.caller);
   };
   this.todo_isnot = function test_todo_isnot(a, b, name) {
     self.todo(a != b, name, "Didn't expect " + a + ", but got it",
               Components.stack.caller);
@@ -1000,18 +1077,19 @@ function testScope(aTester, aTest, expec
     if (typeof(min) != "number" || typeof(max) != "number" ||
         min < 0 || max < min) {
       throw "bad parameter to expectAssertions";
     }
     self.__expectedMinAsserts = min;
     self.__expectedMaxAsserts = max;
   };
 
-  this.setExpected = function test_setExpected() {
-    self.__expected = 'fail';
+  this.setExpectedFailuresForSelfTest = function test_setExpectedFailuresForSelfTest(expectedAllowedFailureCount) {
+    aTest.allowFailure = true;
+    aTest.expectedAllowedFailureCount = expectedAllowedFailureCount;
   };
 
   this.finish = function test_finish() {
     self.__done = true;
     if (self.__waitTimer) {
       self.executeSoon(function() {
         if (self.__done && self.__waitTimer) {
           clearTimeout(self.__waitTimer);
@@ -1032,17 +1110,16 @@ function testScope(aTester, aTest, expec
 testScope.prototype = {
   __done: true,
   __tasks: null,
   __waitTimer: null,
   __cleanupFunctions: [],
   __timeoutFactor: 1,
   __expectedMinAsserts: 0,
   __expectedMaxAsserts: 0,
-  __expected: 'pass',
 
   EventUtils: {},
   SimpleTest: {},
   Task: null,
   ContentTask: null,
   BrowserTestUtils: null,
   TestUtils: null,
   ExtensionTestUtils: null,
--- a/testing/mochitest/tests/browser/browser.ini
+++ b/testing/mochitest/tests/browser/browser.ini
@@ -1,49 +1,39 @@
 [DEFAULT]
 support-files =
   head.js
 
-[browser_browserLoaded_content_loaded.js]
 [browser_add_task.js]
 [browser_async.js]
+[browser_browserLoaded_content_loaded.js]
 [browser_BrowserTestUtils.js]
 support-files =
   dummy.html
+[browser_fail.js]
+[browser_fail_add_task.js]
+[browser_fail_async.js]
+[browser_fail_if.js]
+fail-if = true
+[browser_fail_throw.js]
+[browser_fail_timeout.js]
+skip-if = true # Disabled beacuse it takes too long (bug 1178959)
+[browser_fail_uncaught_rejection.js]
+[browser_fail_unexpectedTimeout.js]
+skip-if = true # Disabled beacuse it takes too long (bug 1178959)
+[browser_getTestFile.js]
+support-files =
+  test-dir/*
+  waitForFocusPage.html
 [browser_head.js]
 [browser_pass.js]
 [browser_parameters.js]
 [browser_popupNode.js]
 [browser_popupNode_check.js]
 [browser_privileges.js]
+[browser_requestLongerTimeout.js]
+skip-if = true # Disabled beacuse it takes too long (bug 1178959)
 [browser_sanityException.js]
 [browser_sanityException2.js]
 [browser_waitForFocus.js]
 skip-if = (os == "win" && e10s && debug)
-[browser_getTestFile.js]
-support-files =
-  test-dir/*
-  waitForFocusPage.html
-
-# Disabled because it would take too long, useful to check functionality though.
-#  browser_requestLongerTimeout.js
 [browser_zz_fail_openwindow.js]
 skip-if = true # this catches outside of the main loop to find an extra window
-[browser_fail.js]
-skip-if = true
-[browser_fail_add_task.js]
-skip-if = true # fail-if doesnt catch an exception outside the test
-[browser_fail_async_throw.js]
-skip-if = true # fail-if doesnt catch an exception outside the test
-[browser_fail_fp.js]
-fail-if = true
-[browser_fail_pf.js]
-fail-if = true
-[browser_fail_throw.js]
-skip-if = true # fail-if doesnt catch an exception outside the test
-
-# Disabled beacuse it takes too long (bug 1178959)
-[browser_fail_timeout.js]
-skip-if = true
-# Disabled beacuse it takes too long (bug 1178959)
-[browser_fail_unexpectedTimeout.js]
-skip-if = true
-
--- a/testing/mochitest/tests/browser/browser_async.js
+++ b/testing/mochitest/tests/browser/browser_async.js
@@ -1,8 +1,8 @@
 function test() {
   waitForExplicitFinish();
   function done() {
     ok(true, "timeout ran");
     finish();
   }
-  setTimeout(done, 10000);
+  setTimeout(done, 500);
 }
--- a/testing/mochitest/tests/browser/browser_fail.js
+++ b/testing/mochitest/tests/browser/browser_fail.js
@@ -1,8 +1,10 @@
+setExpectedFailuresForSelfTest(6);
+
 function test() {
   ok(false, "fail ok");
   is(true, false, "fail is");
   isnot(true, true, "fail isnot");
   todo(true, "fail todo");
   todo_is(true, true, "fail todo_is");
   todo_isnot(true, false, "fail todo_isnot");
 }
--- a/testing/mochitest/tests/browser/browser_fail_add_task.js
+++ b/testing/mochitest/tests/browser/browser_fail_add_task.js
@@ -1,57 +1,27 @@
 /* Any copyright is dedicated to the Public Domain.
  * http://creativecommons.org/publicdomain/zero/1.0/ */
 
 "use strict";
 
-// This test is designed to fail.
-// It ensures that throwing an asynchronous error from add_task will
-// fail the test.
-
-var passedTests = 0;
+setExpectedFailuresForSelfTest(4);
 
-function rejectWithTimeout(error = undefined) {
-  let deferred = Promise.defer();
-  executeSoon(function() {
-    ok(true, "we get here after a timeout");
-    deferred.reject(error);
-  });
-  return deferred.promise;
+function rejectOnNextTick(error) {
+  return new Promise((resolve, reject) => executeSoon(() => reject(error)));
 }
 
 add_task(function failWithoutError() {
-  try {
-    yield rejectWithTimeout();
-  } finally {
-    ++passedTests;
-  }
+  yield rejectOnNextTick(undefined);
 });
 
 add_task(function failWithString() {
-  try {
-    yield rejectWithTimeout("Meaningless error");
-  } finally {
-    ++passedTests;
-  }
+  yield rejectOnNextTick("This is a string");
 });
 
-add_task(function failWithoutInt() {
-  try {
-    yield rejectWithTimeout(42);
-  } finally {
-    ++passedTests;
-  }
+add_task(function failWithInt() {
+  yield rejectOnNextTick(42);
 });
 
-
 // This one should display a stack trace
 add_task(function failWithError() {
-  try {
-    yield rejectWithTimeout(new Error("This is an error"));
-  } finally {
-    ++passedTests;
-  }
+  yield rejectOnNextTick(new Error("This is an error"));
 });
-
-add_task(function done() {
-  is(passedTests, 4, "Passed all tests");
-});
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/browser/browser_fail_async.js
@@ -0,0 +1,9 @@
+setExpectedFailuresForSelfTest(1);
+
+function test() {
+  waitForExplicitFinish();
+  executeSoon(() => {
+    ok(false, "fail");
+    finish();
+  });
+}
deleted file mode 100644
--- a/testing/mochitest/tests/browser/browser_fail_async_throw.js
+++ /dev/null
@@ -1,7 +0,0 @@
-function test() {
-  function end() {
-    throw "thrown exception";
-  }
-  waitForExplicitFinish();
-  setTimeout(end, 1000);
-}
deleted file mode 100644
--- a/testing/mochitest/tests/browser/browser_fail_fp.js
+++ /dev/null
@@ -1,4 +0,0 @@
-function test() {
-  ok(false, "first fail ok");
-  ok(true, "then pass ok");
-}
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/browser/browser_fail_if.js
@@ -0,0 +1,4 @@
+// We expect this test to fail because it is marked as fail-if in the manifest.
+function test() {
+  ok(false, "fail ok");
+}
deleted file mode 100644
--- a/testing/mochitest/tests/browser/browser_fail_pf.js
+++ /dev/null
@@ -1,4 +0,0 @@
-function test() {
-  ok(true, "first pass ok");
-  ok(false, "then fail ok");
-}
--- a/testing/mochitest/tests/browser/browser_fail_throw.js
+++ b/testing/mochitest/tests/browser/browser_fail_throw.js
@@ -1,3 +1,5 @@
+setExpectedFailuresForSelfTest(1);
+
 function test() {
   throw "thrown exception";
 }
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/browser/browser_fail_uncaught_rejection.js
@@ -0,0 +1,6 @@
+setExpectedFailuresForSelfTest(1);
+
+function test() {
+  Components.utils.import("resource://gre/modules/Promise.jsm", this);
+  Promise.reject(new Error("Promise rejection."));
+}