Backed out changeset 6da58c7bb247 (bug 1365649) for frequently failing devtools/client/netmonitor/test/browser_net_simple-request-data.js. r=backout
authorSebastian Hengst <archaeopteryx@coole-files.de>
Sun, 28 May 2017 11:32:16 +0200
changeset 409164 398d167b26f90af9953523ef4cbef3e38af5eb4a
parent 409163 5f54472fcf58d462a31398d15cacbdde5bcad7e1
child 409165 2e61d7b8278567b029369b2dca13a26421c945cb
push id7391
push usermtabara@mozilla.com
push dateMon, 12 Jun 2017 13:08:53 +0000
treeherdermozilla-beta@2191d7f87e2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1365649
milestone55.0a1
backs out6da58c7bb247d3e879012bea8d848eb68f16e36e
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset 6da58c7bb247 (bug 1365649) for frequently failing devtools/client/netmonitor/test/browser_net_simple-request-data.js. r=backout
testing/mochitest/browser-harness.xul
testing/mochitest/browser-test.js
testing/mochitest/tests/browser/browser.ini
testing/mochitest/tests/browser/browser_async.js
testing/mochitest/tests/browser/browser_fail.js
testing/mochitest/tests/browser/browser_fail_add_task.js
testing/mochitest/tests/browser/browser_fail_async.js
testing/mochitest/tests/browser/browser_fail_async_throw.js
testing/mochitest/tests/browser/browser_fail_fp.js
testing/mochitest/tests/browser/browser_fail_if.js
testing/mochitest/tests/browser/browser_fail_pf.js
testing/mochitest/tests/browser/browser_fail_throw.js
testing/mochitest/tests/browser/browser_fail_uncaught_rejection.js
--- a/testing/mochitest/browser-harness.xul
+++ b/testing/mochitest/browser-harness.xul
@@ -127,19 +127,16 @@
         return this.results.filter(t => !t.info && !t.todo && t.pass).length;
       },
       get todoCount() {
         return this.results.filter(t => !t.info && t.todo && t.pass).length;
       },
       get failCount() {
         return this.results.filter(t => !t.info && !t.pass).length;
       },
-      get allowedFailureCount() {
-        return this.results.filter(t => t.allowedFailure).length;
-      },
 
       addResult: function addResult(result) {
         this.lastOutputTime = Date.now();
         this.results.push(result);
 
         if (result.info) {
           if (result.msg) {
             this.dumper.structuredLogger.info(result.msg);
--- a/testing/mochitest/browser-test.js
+++ b/testing/mochitest/browser-test.js
@@ -160,41 +160,44 @@ function Tester(aTests, structuredLogger
   SIMPLETEST_OVERRIDES.forEach(m => {
     this.SimpleTestOriginal[m] = this.SimpleTest[m];
   });
 
   this._coverageCollector = null;
 
   this._toleratedUncaughtRejections = null;
   this._uncaughtErrorObserver = ({message, date, fileName, stack, lineNumber}) => {
-    let ex = message;
+    let error = message;
     if (fileName || lineNumber) {
-      ex = {
+      error = {
         fileName: fileName,
         lineNumber: lineNumber,
         message: message,
         toString: function() {
           return message;
         }
       };
     }
 
     // We may have a whitelist of rejections we wish to tolerate.
-    let pass = this._toleratedUncaughtRejections &&
+    let tolerate = this._toleratedUncaughtRejections &&
       this._toleratedUncaughtRejections.indexOf(message) != -1;
     let name = "A promise chain failed to handle a rejection: ";
-    if (pass) {
+    if (tolerate) {
       name = "WARNING: (PLEASE FIX THIS AS PART OF BUG 1077403) " + name;
     }
 
-    this.currentTest.addResult(new testResult({
-      pass, name, ex, todo: pass, stack,
-      allowFailure: this.currentTest.allowFailure,
-    }));
-  };
+    this.currentTest.addResult(
+      new testResult(
+	      /*success*/tolerate,
+        /*name*/name,
+        /*error*/error,
+        /*known*/tolerate,
+        /*stack*/stack));
+    };
 }
 Tester.prototype = {
   EventUtils: {},
   SimpleTest: {},
   Task: null,
   ContentTask: null,
   ExtensionTestUtils: null,
   Assert: null,
@@ -273,21 +276,19 @@ Tester.prototype = {
     let baseMsg = timedOut ? "Found a {elt} after previous test timed out"
                            : this.currentTest ? "Found an unexpected {elt} at the end of test run"
                                               : "Found an unexpected {elt}";
 
     // Remove stale tabs
     if (this.currentTest && window.gBrowser && gBrowser.tabs.length > 1) {
       while (gBrowser.tabs.length > 1) {
         let lastTab = gBrowser.tabContainer.lastChild;
-        this.currentTest.addResult(new testResult({
-          name: baseMsg.replace("{elt}", "tab") + ": " +
-                lastTab.linkedBrowser.currentURI.spec,
-          allowFailure: this.currentTest.allowFailure,
-        }));
+        let msg = baseMsg.replace("{elt}", "tab") +
+                  ": " + lastTab.linkedBrowser.currentURI.spec;
+        this.currentTest.addResult(new testResult(false, msg, "", false));
         gBrowser.removeTab(lastTab);
       }
     }
 
     // Replace the last tab with a fresh one
     if (window.gBrowser) {
       let newTab = gBrowser.addTab("about:blank", { skipAnimation: true });
       gBrowser.removeTab(gBrowser.selectedTab, { skipPermitUnload: true });
@@ -309,20 +310,17 @@ Tester.prototype = {
           break;
         case null:
           type = "unknown window with document URI: " + win.document.documentURI +
                  " and title: " + win.document.title;
           break;
         }
         let msg = baseMsg.replace("{elt}", type);
         if (this.currentTest) {
-          this.currentTest.addResult(new testResult({
-            name: msg,
-            allowFailure: this.currentTest.allowFailure,
-          }));
+          this.currentTest.addResult(new testResult(false, msg, "", false));
         } else {
           if (!createdFakeTestForLogging) {
             createdFakeTestForLogging = true;
             this.structuredLogger.testStart("browser-test.js");
           }
           this.failuresFromInitialWindowState++;
           this.structuredLogger.testStatus("browser-test.js",
                                            msg, "FAIL", false, "");
@@ -431,69 +429,55 @@ Tester.prototype = {
       // next one.
       let testScope = this.currentTest.scope;
       while (testScope.__cleanupFunctions.length > 0) {
         let func = testScope.__cleanupFunctions.shift();
         try {
           yield func.apply(testScope);
         }
         catch (ex) {
-          this.currentTest.addResult(new testResult({
-            name: "Cleanup function threw an exception",
-            ex,
-            allowFailure: this.currentTest.allowFailure,
-          }));
+          this.currentTest.addResult(new testResult(false, "Cleanup function threw an exception", ex, false));
         }
       }
 
       if (this.currentTest.passCount === 0 &&
           this.currentTest.failCount === 0 &&
           this.currentTest.todoCount === 0) {
-        this.currentTest.addResult(new testResult({
-          name: "This test contains no passes, no fails and no todos. Maybe" +
-                " it threw a silent exception? Make sure you use" +
-                " waitForExplicitFinish() if you need it.",
-        }));
+        this.currentTest.addResult(new testResult(false, "This test contains no passes, no fails and no todos. Maybe it threw a silent exception? Make sure you use waitForExplicitFinish() if you need it.", "", false));
+      }
+
+      if (testScope.__expected == 'fail' && testScope.__num_failed <= 0) {
+        this.currentTest.addResult(new testResult(false, "We expected at least one assertion to fail because this test file was marked as fail-if in the manifest!", "", true));
       }
 
       this.Promise.Debugging.flushUncaughtErrors();
 
       let winUtils = window.QueryInterface(Ci.nsIInterfaceRequestor)
                            .getInterface(Ci.nsIDOMWindowUtils);
       if (winUtils.isTestControllingRefreshes) {
-        this.currentTest.addResult(new testResult({
-          name: "test left refresh driver under test control",
-        }));
+        this.currentTest.addResult(new testResult(false, "test left refresh driver under test control", "", false));
         winUtils.restoreNormalRefresh();
       }
 
       if (this.SimpleTest.isExpectingUncaughtException()) {
-        this.currentTest.addResult(new testResult({
-          name: "expectUncaughtException was called but no uncaught" +
-                " exception was detected!",
-          allowFailure: this.currentTest.allowFailure,
-        }));
+        this.currentTest.addResult(new testResult(false, "expectUncaughtException was called but no uncaught exception was detected!", "", false));
       }
 
       Object.keys(window).forEach(function (prop) {
         if (parseInt(prop) == prop) {
           // This is a string which when parsed as an integer and then
           // stringified gives the original string.  As in, this is in fact a
           // string representation of an integer, so an index into
           // window.frames.  Skip those.
           return;
         }
         if (this._globalProperties.indexOf(prop) == -1) {
           this._globalProperties.push(prop);
-          if (this._globalPropertyWhitelist.indexOf(prop) == -1) {
-            this.currentTest.addResult(new testResult({
-              name: "test left unexpected property on window: " + prop,
-              allowFailure: this.currentTest.allowFailure,
-            }));
-          }
+          if (this._globalPropertyWhitelist.indexOf(prop) == -1)
+            this.currentTest.addResult(new testResult(false, "test left unexpected property on window: " + prop, "", false));
         }
       }, this);
 
       // Clear document.popupNode.  The test could have set it to a custom value
       // for its own purposes, nulling it out it will go back to the default
       // behavior of returning the last opened popup.
       document.popupNode = null;
 
@@ -513,82 +497,52 @@ Tester.prototype = {
               this.structuredLogger.info(msg);
             }
           }
         }
       }
 
       // Notify a long running test problem if it didn't end up in a timeout.
       if (this.currentTest.unexpectedTimeouts && !this.currentTest.timedOut) {
-        this.currentTest.addResult(new testResult({
-          name: "This test exceeded the timeout threshold. It should be" +
-                " rewritten or split up. If that's not possible, use" +
-                " requestLongerTimeout(N), but only as a last resort.",
-        }));
+        let msg = "This test exceeded the timeout threshold. It should be " +
+                  "rewritten or split up. If that's not possible, use " +
+                  "requestLongerTimeout(N), but only as a last resort.";
+        this.currentTest.addResult(new testResult(false, msg, "", false));
       }
 
       // If we're in a debug build, check assertion counts.  This code
       // is similar to the code in TestRunner.testUnloaded in
       // TestRunner.js used for all other types of mochitests.
       let debugsvc = Cc["@mozilla.org/xpcom/debug;1"].getService(Ci.nsIDebug2);
       if (debugsvc.isDebugBuild) {
         let newAssertionCount = debugsvc.assertionCount;
         let numAsserts = newAssertionCount - this.lastAssertionCount;
         this.lastAssertionCount = newAssertionCount;
 
         let max = testScope.__expectedMaxAsserts;
         let min = testScope.__expectedMinAsserts;
         if (numAsserts > max) {
-          // TEST-UNEXPECTED-FAIL
-          this.currentTest.addResult(new testResult({
-            name: "Assertion count " + numAsserts +
-                  " is greater than expected range " +
-                  min + "-" + max + " assertions.",
-            pass: true, // TEMPORARILY TEST-KNOWN-FAIL
-            todo: true,
-            allowFailure: this.currentTest.allowFailure,
-          }));
+          let msg = "Assertion count " + numAsserts +
+                    " is greater than expected range " +
+                    min + "-" + max + " assertions.";
+          // TEST-UNEXPECTED-FAIL (TEMPORARILY TEST-KNOWN-FAIL)
+          //this.currentTest.addResult(new testResult(false, msg, "", false));
+          this.currentTest.addResult(new testResult(true, msg, "", true));
         } else if (numAsserts < min) {
+          let msg = "Assertion count " + numAsserts +
+                    " is less than expected range " +
+                    min + "-" + max + " assertions.";
           // TEST-UNEXPECTED-PASS
-          this.currentTest.addResult(new testResult({
-            name: "Assertion count " + numAsserts +
-                  " is less than expected range " +
-                  min + "-" + max + " assertions.",
-            todo: true,
-            allowFailure: this.currentTest.allowFailure,
-          }));
+          this.currentTest.addResult(new testResult(false, msg, "", true));
         } else if (numAsserts > 0) {
+          let msg = "Assertion count " + numAsserts +
+                    " is within expected range " +
+                    min + "-" + max + " assertions.";
           // TEST-KNOWN-FAIL
-          this.currentTest.addResult(new testResult({
-            name: "Assertion count " + numAsserts +
-                  " is within expected range " +
-                  min + "-" + max + " assertions.",
-            pass: true,
-            todo: true,
-            allowFailure: this.currentTest.allowFailure,
-          }));
-        }
-      }
-
-      if (this.currentTest.allowFailure) {
-        if (this.currentTest.expectedAllowedFailureCount) {
-          this.currentTest.addResult(new testResult({
-            name: "Expected " +
-                  this.currentTest.expectedAllowedFailureCount +
-                  " failures in this file, got " +
-                  this.currentTest.allowedFailureCount + ".",
-            pass: this.currentTest.expectedAllowedFailureCount ==
-                  this.currentTest.allowedFailureCount,
-          }));
-        } else if (this.currentTest.allowedFailureCount == 0) {
-          this.currentTest.addResult(new testResult({
-            name: "We expect at least one assertion to fail because this" +
-                  " test file is marked as fail-if in the manifest.",
-            todo: true,
-          }));
+          this.currentTest.addResult(new testResult(true, msg, "", true));
         }
       }
 
       // Dump memory stats for main thread.
       if (Cc["@mozilla.org/xre/runtime;1"]
           .getService(Ci.nsIXULRuntime)
           .processType == Ci.nsIXULRuntime.PROCESS_TYPE_DEFAULT)
       {
@@ -730,27 +684,23 @@ Tester.prototype = {
     this.currentTest.scope.gTestPath = this.currentTest.path;
     this.currentTest.scope.Task = this.Task;
     this.currentTest.scope.ContentTask = this.ContentTask;
     this.currentTest.scope.BrowserTestUtils = this.BrowserTestUtils;
     this.currentTest.scope.TestUtils = this.TestUtils;
     this.currentTest.scope.ExtensionTestUtils = this.ExtensionTestUtils;
     // Pass a custom report function for mochitest style reporting.
     this.currentTest.scope.Assert = new this.Assert(function(err, message, stack) {
-      currentTest.addResult(new testResult(err ? {
-        name: err.message,
-        ex: err.stack,
-        stack: err.stack,
-        allowFailure: currentTest.allowFailure,
-      } : {
-        name: message,
-        pass: true,
-        stack,
-        allowFailure: currentTest.allowFailure,
-      }));
+      let res;
+      if (err) {
+        res = new testResult(false, err.message, err.stack, false, err.stack);
+      } else {
+        res = new testResult(true, message, "", false, stack);
+      }
+      currentTest.addResult(res);
     });
 
     this.ContentTask.setTestScope(currentScope);
 
     // Allow Assert.jsm methods to be tacked to the current scope.
     this.currentTest.scope.export_assertions = function() {
       for (let func in this.Assert) {
         this[func] = this.Assert[func].bind(this.Assert);
@@ -773,20 +723,17 @@ Tester.prototype = {
     var headPath = currentTestDirPath + "/head.js";
     try {
       this._scriptLoader.loadSubScript(headPath, this.currentTest.scope);
     } catch (ex) {
       // Ignore if no head.js exists, but report all other errors.  Note this
       // will also ignore an existing head.js attempting to import a missing
       // module - see bug 755558 for why this strategy is preferred anyway.
       if (!/^Error opening input stream/.test(ex.toString())) {
-       this.currentTest.addResult(new testResult({
-         name: "head.js import threw an exception",
-         ex,
-       }));
+       this.currentTest.addResult(new testResult(false, "head.js import threw an exception", ex, false));
       }
     }
 
     // Import the test script.
     try {
       this._scriptLoader.loadSubScript(this.currentTest.path,
                                        this.currentTest.scope);
       this.Promise.Debugging.flushUncaughtErrors();
@@ -800,42 +747,36 @@ Tester.prototype = {
         let Promise = this.Promise;
         this.Task.spawn(function*() {
           let task;
           while ((task = this.__tasks.shift())) {
             this.SimpleTest.info("Entering test " + task.name);
             try {
               yield task();
             } catch (ex) {
-              currentTest.addResult(new testResult({
-                name: "Uncaught exception",
-                pass: this.SimpleTest.isExpectingUncaughtException(),
-                ex,
-                stack: (typeof ex == "object" && "stack" in ex) ? ex.stack : null,
-                allowFailure: currentTest.allowFailure,
-              }));
+              let isExpected = !!this.SimpleTest.isExpectingUncaughtException();
+              let stack = (typeof ex == "object" && "stack" in ex)?ex.stack:null;
+              let name = "Uncaught exception";
+              let result = new testResult(isExpected, name, ex, false, stack);
+              currentTest.addResult(result);
             }
             Promise.Debugging.flushUncaughtErrors();
             this.SimpleTest.info("Leaving test " + task.name);
           }
           this.finish();
         }.bind(currentScope));
       } else if (typeof this.currentTest.scope.test == "function") {
         this.currentTest.scope.test();
       } else {
         throw "This test didn't call add_task, nor did it define a generatorTest() function, nor did it define a test() function, so we don't know how to run it.";
       }
     } catch (ex) {
+      let isExpected = !!this.SimpleTest.isExpectingUncaughtException();
       if (!this.SimpleTest.isIgnoringAllUncaughtExceptions()) {
-        this.currentTest.addResult(new testResult({
-          name: "Exception thrown",
-          pass: this.SimpleTest.isExpectingUncaughtException(),
-          ex,
-          allowFailure: this.currentTest.allowFailure,
-        }));
+        this.currentTest.addResult(new testResult(isExpected, "Exception thrown", ex, false));
         this.SimpleTest.expectUncaughtException(false);
       } else {
         this.currentTest.addResult(new testMessage("Exception thrown: " + ex));
       }
       this.currentTest.scope.finish();
     }
 
     // If the test ran synchronously, move to the next test, otherwise the test
@@ -878,141 +819,123 @@ Tester.prototype = {
         const MAX_UNEXPECTED_TIMEOUTS = 10;
         if (Date.now() - self.currentTest.lastOutputTime < (gTimeoutSeconds / 2) * 1000 &&
             ++self.currentTest.unexpectedTimeouts <= MAX_UNEXPECTED_TIMEOUTS) {
             self.currentTest.scope.__waitTimer =
               setTimeout(timeoutFn, gTimeoutSeconds * 1000);
           return;
         }
 
-        self.currentTest.addResult(new testResult({ name: "Test timed out" }));
+        self.currentTest.addResult(new testResult(false, "Test timed out", null, false));
         self.currentTest.timedOut = true;
         self.currentTest.scope.__waitTimer = null;
         self.nextTest();
       }, gTimeoutSeconds * 1000]);
     }
   },
 
   QueryInterface: function(aIID) {
     if (aIID.equals(Ci.nsIConsoleListener) ||
         aIID.equals(Ci.nsISupports))
       return this;
 
     throw Components.results.NS_ERROR_NO_INTERFACE;
   }
 };
 
-/**
- * Represents the result of one test assertion. This is described with a string
- * in traditional logging, and has a "status" and "expected" property used in
- * structured logging. Normally, results are mapped as follows:
- *
- *   pass:    todo:    Added to:    Described as:           Status:  Expected:
- *     true     false    passCount    TEST-PASS               PASS     PASS
- *     true     true     todoCount    TEST-KNOWN-FAIL         FAIL     FAIL
- *     false    false    failCount    TEST-UNEXPECTED-FAIL    FAIL     PASS
- *     false    true     failCount    TEST-UNEXPECTED-PASS    PASS     FAIL
- *
- * The "allowFailure" argument indicates that this is one of the assertions that
- * should be allowed to fail, for example because "fail-if" is true for the
- * current test file in the manifest. In this case, results are mapped this way:
- *
- *   pass:    todo:    Added to:    Described as:           Status:  Expected:
- *     true     false    passCount    TEST-PASS               PASS     PASS
- *     true     true     todoCount    TEST-KNOWN-FAIL         FAIL     FAIL
- *     false    false    todoCount    TEST-KNOWN-FAIL         FAIL     FAIL
- *     false    true     todoCount    TEST-KNOWN-FAIL         FAIL     FAIL
- */
-function testResult({ name, pass, todo, ex, stack, allowFailure }) {
-  this.info = false;
-  this.name = name;
+function testResult(aCondition, aName, aDiag, aIsTodo, aStack) {
+  this.name = aName;
   this.msg = "";
 
-  if (allowFailure && !pass) {
-    this.allowedFailure = true;
-    this.pass = true;
-    this.todo = true;
-  } else {
-    this.pass = !!pass;
-    this.todo = todo;
-  }
-
-  this.expected = this.todo ? "FAIL" : "PASS";
+  this.info = false;
+  this.pass = !!aCondition;
+  this.todo = aIsTodo;
 
   if (this.pass) {
-    this.status = this.expected;
-    return;
-  }
-
-  this.status = this.todo ? "PASS" : "FAIL";
-
-  if (ex) {
-    if (typeof ex == "object" && "fileName" in ex) {
-      // we have an exception - print filename and linenumber information
-      this.msg += "at " + ex.fileName + ":" + ex.lineNumber + " - ";
+    if (aIsTodo) {
+      this.status = "FAIL";
+      this.expected = "FAIL";
+    } else {
+      this.status = "PASS";
+      this.expected = "PASS";
     }
-    this.msg += String(ex);
-  }
 
-  if (stack) {
-    this.msg += "\nStack trace:\n";
-    let normalized;
-    if (stack instanceof Components.interfaces.nsIStackFrame) {
-      let frames = [];
-      for (let frame = stack; frame; frame = frame.caller) {
-        frames.push(frame.filename + ":" + frame.name + ":" + frame.lineNumber);
+  } else {
+    if (aDiag) {
+      if (typeof aDiag == "object" && "fileName" in aDiag) {
+        // we have an exception - print filename and linenumber information
+        this.msg += "at " + aDiag.fileName + ":" + aDiag.lineNumber + " - ";
       }
-      normalized = frames.join("\n");
+      this.msg += String(aDiag);
+    }
+    if (aStack) {
+      this.msg += "\nStack trace:\n";
+      let normalized;
+      if (aStack instanceof Components.interfaces.nsIStackFrame) {
+        let frames = [];
+        for (let frame = aStack; frame; frame = frame.caller) {
+          frames.push(frame.filename + ":" + frame.name + ":" + frame.lineNumber);
+        }
+        normalized = frames.join("\n");
+      } else {
+        normalized = "" + aStack;
+      }
+      this.msg += Task.Debugging.generateReadableStack(normalized, "    ");
+    }
+    if (aIsTodo) {
+      this.status = "PASS";
+      this.expected = "FAIL";
     } else {
-      normalized = "" + stack;
+      this.status = "FAIL";
+      this.expected = "PASS";
     }
-    this.msg += Task.Debugging.generateReadableStack(normalized, "    ");
-  }
 
-  if (gConfig.debugOnFailure) {
-    // You've hit this line because you requested to break into the
-    // debugger upon a testcase failure on your test run.
-    debugger;
+    if (gConfig.debugOnFailure) {
+      // You've hit this line because you requested to break into the
+      // debugger upon a testcase failure on your test run.
+      debugger;
+    }
   }
 }
 
-function testMessage(msg) {
-  this.msg = msg || "";
+function testMessage(aName) {
+  this.msg = aName || "";
   this.info = true;
 }
 
 // Need to be careful adding properties to this object, since its properties
 // cannot conflict with global variables used in tests.
 function testScope(aTester, aTest, expected) {
   this.__tester = aTester;
-
-  aTest.allowFailure = expected == "fail";
+  this.__expected = expected;
+  this.__num_failed = 0;
 
   var self = this;
-  this.ok = function test_ok(condition, name, ex, stack) {
-    aTest.addResult(new testResult({
-      name, pass: condition, ex,
-      stack: stack || Components.stack.caller,
-      allowFailure: aTest.allowFailure,
-    }));
+  this.ok = function test_ok(condition, name, diag, stack) {
+    if (self.__expected == 'fail') {
+        if (!condition) {
+          self.__num_failed++;
+          condition = true;
+        }
+    }
+
+    aTest.addResult(new testResult(condition, name, diag, false,
+                                   stack ? stack : Components.stack.caller));
   };
   this.is = function test_is(a, b, name) {
     self.ok(a == b, name, "Got " + a + ", expected " + b, false,
             Components.stack.caller);
   };
   this.isnot = function test_isnot(a, b, name) {
     self.ok(a != b, name, "Didn't expect " + a + ", but got it", false,
             Components.stack.caller);
   };
-  this.todo = function test_todo(condition, name, ex, stack) {
-    aTest.addResult(new testResult({
-      name, pass: !condition, todo: true, ex,
-      stack: stack || Components.stack.caller,
-      allowFailure: aTest.allowFailure,
-    }));
+  this.todo = function test_todo(condition, name, diag, stack) {
+    aTest.addResult(new testResult(!condition, name, diag, true,
+                                   stack ? stack : Components.stack.caller));
   };
   this.todo_is = function test_todo_is(a, b, name) {
     self.todo(a == b, name, "Got " + a + ", expected " + b,
               Components.stack.caller);
   };
   this.todo_isnot = function test_todo_isnot(a, b, name) {
     self.todo(a != b, name, "Didn't expect " + a + ", but got it",
               Components.stack.caller);
@@ -1077,19 +1000,18 @@ function testScope(aTester, aTest, expec
     if (typeof(min) != "number" || typeof(max) != "number" ||
         min < 0 || max < min) {
       throw "bad parameter to expectAssertions";
     }
     self.__expectedMinAsserts = min;
     self.__expectedMaxAsserts = max;
   };
 
-  this.setExpectedFailuresForSelfTest = function test_setExpectedFailuresForSelfTest(expectedAllowedFailureCount) {
-    aTest.allowFailure = true;
-    aTest.expectedAllowedFailureCount = expectedAllowedFailureCount;
+  this.setExpected = function test_setExpected() {
+    self.__expected = 'fail';
   };
 
   this.finish = function test_finish() {
     self.__done = true;
     if (self.__waitTimer) {
       self.executeSoon(function() {
         if (self.__done && self.__waitTimer) {
           clearTimeout(self.__waitTimer);
@@ -1110,16 +1032,17 @@ function testScope(aTester, aTest, expec
 testScope.prototype = {
   __done: true,
   __tasks: null,
   __waitTimer: null,
   __cleanupFunctions: [],
   __timeoutFactor: 1,
   __expectedMinAsserts: 0,
   __expectedMaxAsserts: 0,
+  __expected: 'pass',
 
   EventUtils: {},
   SimpleTest: {},
   Task: null,
   ContentTask: null,
   BrowserTestUtils: null,
   TestUtils: null,
   ExtensionTestUtils: null,
--- a/testing/mochitest/tests/browser/browser.ini
+++ b/testing/mochitest/tests/browser/browser.ini
@@ -1,39 +1,49 @@
 [DEFAULT]
 support-files =
   head.js
 
+[browser_browserLoaded_content_loaded.js]
 [browser_add_task.js]
 [browser_async.js]
-[browser_browserLoaded_content_loaded.js]
 [browser_BrowserTestUtils.js]
 support-files =
   dummy.html
-[browser_fail.js]
-[browser_fail_add_task.js]
-[browser_fail_async.js]
-[browser_fail_if.js]
-fail-if = true
-[browser_fail_throw.js]
-[browser_fail_timeout.js]
-skip-if = true # Disabled beacuse it takes too long (bug 1178959)
-[browser_fail_uncaught_rejection.js]
-[browser_fail_unexpectedTimeout.js]
-skip-if = true # Disabled beacuse it takes too long (bug 1178959)
-[browser_getTestFile.js]
-support-files =
-  test-dir/*
-  waitForFocusPage.html
 [browser_head.js]
 [browser_pass.js]
 [browser_parameters.js]
 [browser_popupNode.js]
 [browser_popupNode_check.js]
 [browser_privileges.js]
-[browser_requestLongerTimeout.js]
-skip-if = true # Disabled beacuse it takes too long (bug 1178959)
 [browser_sanityException.js]
 [browser_sanityException2.js]
 [browser_waitForFocus.js]
 skip-if = (os == "win" && e10s && debug)
+[browser_getTestFile.js]
+support-files =
+  test-dir/*
+  waitForFocusPage.html
+
+# Disabled because it would take too long, useful to check functionality though.
+#  browser_requestLongerTimeout.js
 [browser_zz_fail_openwindow.js]
 skip-if = true # this catches outside of the main loop to find an extra window
+[browser_fail.js]
+skip-if = true
+[browser_fail_add_task.js]
+skip-if = true # fail-if doesnt catch an exception outside the test
+[browser_fail_async_throw.js]
+skip-if = true # fail-if doesnt catch an exception outside the test
+[browser_fail_fp.js]
+fail-if = true
+[browser_fail_pf.js]
+fail-if = true
+[browser_fail_throw.js]
+skip-if = true # fail-if doesnt catch an exception outside the test
+
+# Disabled beacuse it takes too long (bug 1178959)
+[browser_fail_timeout.js]
+skip-if = true
+# Disabled beacuse it takes too long (bug 1178959)
+[browser_fail_unexpectedTimeout.js]
+skip-if = true
+
--- a/testing/mochitest/tests/browser/browser_async.js
+++ b/testing/mochitest/tests/browser/browser_async.js
@@ -1,8 +1,8 @@
 function test() {
   waitForExplicitFinish();
   function done() {
     ok(true, "timeout ran");
     finish();
   }
-  setTimeout(done, 500);
+  setTimeout(done, 10000);
 }
--- a/testing/mochitest/tests/browser/browser_fail.js
+++ b/testing/mochitest/tests/browser/browser_fail.js
@@ -1,10 +1,8 @@
-setExpectedFailuresForSelfTest(6);
-
 function test() {
   ok(false, "fail ok");
   is(true, false, "fail is");
   isnot(true, true, "fail isnot");
   todo(true, "fail todo");
   todo_is(true, true, "fail todo_is");
   todo_isnot(true, false, "fail todo_isnot");
 }
--- a/testing/mochitest/tests/browser/browser_fail_add_task.js
+++ b/testing/mochitest/tests/browser/browser_fail_add_task.js
@@ -1,27 +1,57 @@
 /* Any copyright is dedicated to the Public Domain.
  * http://creativecommons.org/publicdomain/zero/1.0/ */
 
 "use strict";
 
-setExpectedFailuresForSelfTest(4);
+// This test is designed to fail.
+// It ensures that throwing an asynchronous error from add_task will
+// fail the test.
+
+var passedTests = 0;
 
-function rejectOnNextTick(error) {
-  return new Promise((resolve, reject) => executeSoon(() => reject(error)));
+function rejectWithTimeout(error = undefined) {
+  let deferred = Promise.defer();
+  executeSoon(function() {
+    ok(true, "we get here after a timeout");
+    deferred.reject(error);
+  });
+  return deferred.promise;
 }
 
 add_task(function failWithoutError() {
-  yield rejectOnNextTick(undefined);
+  try {
+    yield rejectWithTimeout();
+  } finally {
+    ++passedTests;
+  }
 });
 
 add_task(function failWithString() {
-  yield rejectOnNextTick("This is a string");
+  try {
+    yield rejectWithTimeout("Meaningless error");
+  } finally {
+    ++passedTests;
+  }
 });
 
-add_task(function failWithInt() {
-  yield rejectOnNextTick(42);
+add_task(function failWithoutInt() {
+  try {
+    yield rejectWithTimeout(42);
+  } finally {
+    ++passedTests;
+  }
 });
 
+
 // This one should display a stack trace
 add_task(function failWithError() {
-  yield rejectOnNextTick(new Error("This is an error"));
+  try {
+    yield rejectWithTimeout(new Error("This is an error"));
+  } finally {
+    ++passedTests;
+  }
 });
+
+add_task(function done() {
+  is(passedTests, 4, "Passed all tests");
+});
deleted file mode 100644
--- a/testing/mochitest/tests/browser/browser_fail_async.js
+++ /dev/null
@@ -1,9 +0,0 @@
-setExpectedFailuresForSelfTest(1);
-
-function test() {
-  waitForExplicitFinish();
-  executeSoon(() => {
-    ok(false, "fail");
-    finish();
-  });
-}
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/browser/browser_fail_async_throw.js
@@ -0,0 +1,7 @@
+function test() {
+  function end() {
+    throw "thrown exception";
+  }
+  waitForExplicitFinish();
+  setTimeout(end, 1000);
+}
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/browser/browser_fail_fp.js
@@ -0,0 +1,4 @@
+function test() {
+  ok(false, "first fail ok");
+  ok(true, "then pass ok");
+}
deleted file mode 100644
--- a/testing/mochitest/tests/browser/browser_fail_if.js
+++ /dev/null
@@ -1,4 +0,0 @@
-// We expect this test to fail because it is marked as fail-if in the manifest.
-function test() {
-  ok(false, "fail ok");
-}
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/browser/browser_fail_pf.js
@@ -0,0 +1,4 @@
+function test() {
+  ok(true, "first pass ok");
+  ok(false, "then fail ok");
+}
--- a/testing/mochitest/tests/browser/browser_fail_throw.js
+++ b/testing/mochitest/tests/browser/browser_fail_throw.js
@@ -1,5 +1,3 @@
-setExpectedFailuresForSelfTest(1);
-
 function test() {
   throw "thrown exception";
 }
deleted file mode 100644
--- a/testing/mochitest/tests/browser/browser_fail_uncaught_rejection.js
+++ /dev/null
@@ -1,6 +0,0 @@
-setExpectedFailuresForSelfTest(1);
-
-function test() {
-  Components.utils.import("resource://gre/modules/Promise.jsm", this);
-  Promise.reject(new Error("Promise rejection."));
-}