Backed out 5 changesets (bug 1392391) for frequently failing jsreftests on Android. a=backout
authorGurzau Raul <rgurzau@mozilla.com>
Wed, 07 Feb 2018 02:20:38 +0200
changeset 402641 e1954b02d9e39bdb7c1f17aa95ca9cad5d5c14ae
parent 402640 f033d62a90ad5524938a329b2a882b4b64e59ccb
child 402659 2bd611e4debb29ff8dea29a90190f2bead634ba6
child 402682 e66bde698178637b8874ea7eede1a4762852a144
push id33395
push userrgurzau@mozilla.com
push dateWed, 07 Feb 2018 00:21:04 +0000
treeherdermozilla-central@e1954b02d9e3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1392391
milestone60.0a1
backs out50df56a0cebf11a8e05bf1d49c678521fee522c6
406806a088d5d10f68a87c9be5c2adce91fd1a9e
6bd3abc55ea80254cfeb5f5450090d602288f1c9
846d50ea84035f3c79b45b0556c52bf4d1ea897d
b9e9374af9a13da41ccec647012182fcaecb4a7a
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 5 changesets (bug 1392391) for frequently failing jsreftests on Android. a=backout Backed out changeset 50df56a0cebf (bug 1392391) Backed out changeset 406806a088d5 (bug 1392391) Backed out changeset 6bd3abc55ea8 (bug 1392391) Backed out changeset 846d50ea8403 (bug 1392391) Backed out changeset b9e9374af9a1 (bug 1392391)
layout/tools/reftest/globals.jsm
layout/tools/reftest/manifest.jsm
layout/tools/reftest/output.py
layout/tools/reftest/reftest.jsm
layout/tools/reftest/remotereftest.py
layout/tools/reftest/runreftest.py
testing/mozbase/mozlog/mozlog/structuredlog.py
--- a/layout/tools/reftest/globals.jsm
+++ b/layout/tools/reftest/globals.jsm
@@ -137,17 +137,16 @@ for (let [key, val] of Object.entries({
     failedDisplayList: false,
     failedOpaqueLayer: false,
     failedOpaqueLayerMessages: [],
     failedAssignedLayer: false,
     failedAssignedLayerMessages: [],
 
     startAfter: undefined,
     suiteStarted: false,
-    manageSuite: false,
 
     // The enabled-state of the test-plugins, stored so they can be reset later
     testPluginEnabledStates: null,
     prefsToRestore: [],
     httpServerPort: -1,
 
     // whether to run slow tests or not
     runSlowTests: true,
--- a/layout/tools/reftest/manifest.jsm
+++ b/layout/tools/reftest/manifest.jsm
@@ -1,16 +1,16 @@
 /* -*- indent-tabs-mode: nil; js-indent-level: 4 -*- /
 /* vim: set shiftwidth=4 tabstop=8 autoindent cindent expandtab: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 "use strict";
 
-this.EXPORTED_SYMBOLS = ["ReadTopManifest", "CreateUrls"];
+this.EXPORTED_SYMBOLS = ["ReadTopManifest"];
 
 var CC = Components.classes;
 const CI = Components.interfaces;
 const CU = Components.utils;
 
 CU.import("chrome://reftest/content/globals.jsm", this);
 CU.import("chrome://reftest/content/reftest.jsm", this);
 CU.import("resource://gre/modules/Services.jsm");
@@ -291,53 +291,103 @@ function ReadManifest(aURL, aFilter)
                     throw "Error in manifest file parsing code: we should never get expected_status=" + expected_status + " when nonSkipUsed=false (from " + aURL.spec + " line " + lineNo + ")";
                 }
 
                 var incURI = g.ioService.newURI(items[1], null, listURL);
                 secMan.checkLoadURIWithPrincipal(principal, incURI,
                                                  CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
                 ReadManifest(incURI, aFilter);
             }
-        } else if (items[0] == TYPE_LOAD || items[0] == TYPE_SCRIPT) {
+        } else if (items[0] == TYPE_LOAD) {
             if (items.length != 2)
-                throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to " + items[0];
-            if (items[0] == TYPE_LOAD && expected_status != EXPECTED_PASS && expected_status != EXPECTED_DEATH)
+                throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to load";
+            if (expected_status != EXPECTED_PASS &&
+                expected_status != EXPECTED_DEATH)
                 throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect known failure type for load test";
+            var [testURI] = runHttp
+                            ? ServeFiles(principal, httpDepth,
+                                         listURL, [items[1]])
+                            : [g.ioService.newURI(items[1], null, listURL)];
+            var prettyPath = runHttp
+                           ? g.ioService.newURI(items[1], null, listURL).spec
+                           : testURI.spec;
+            secMan.checkLoadURIWithPrincipal(principal, testURI,
+                                             CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
             AddTestItem({ type: TYPE_LOAD,
                           expected: expected_status,
-                          manifest: aURL.spec,
                           allowSilentFail: allow_silent_fail,
+                          prettyPath: prettyPath,
                           minAsserts: minAsserts,
                           maxAsserts: maxAsserts,
                           needsFocus: needs_focus,
                           slow: slow,
                           prefSettings1: testPrefSettings,
                           prefSettings2: refPrefSettings,
                           fuzzyMinDelta: fuzzy_delta.min,
                           fuzzyMaxDelta: fuzzy_delta.max,
                           fuzzyMinPixels: fuzzy_pixels.min,
                           fuzzyMaxPixels: fuzzy_pixels.max,
-                          runHttp: runHttp,
-                          httpDepth: httpDepth,
-                          url1: items[1],
+                          url1: testURI,
+                          url2: null,
+                          chaosMode: chaosMode }, aFilter);
+        } else if (items[0] == TYPE_SCRIPT) {
+            if (items.length != 2)
+                throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to script";
+            var [testURI] = runHttp
+                            ? ServeFiles(principal, httpDepth,
+                                         listURL, [items[1]])
+                            : [g.ioService.newURI(items[1], null, listURL)];
+            var prettyPath = runHttp
+                           ? g.ioService.newURI(items[1], null, listURL).spec
+                           : testURI.spec;
+            secMan.checkLoadURIWithPrincipal(principal, testURI,
+                                             CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
+            AddTestItem({ type: TYPE_SCRIPT,
+                          expected: expected_status,
+                          allowSilentFail: allow_silent_fail,
+                          prettyPath: prettyPath,
+                          minAsserts: minAsserts,
+                          maxAsserts: maxAsserts,
+                          needsFocus: needs_focus,
+                          slow: slow,
+                          prefSettings1: testPrefSettings,
+                          prefSettings2: refPrefSettings,
+                          fuzzyMinDelta: fuzzy_delta.min,
+                          fuzzyMaxDelta: fuzzy_delta.max,
+                          fuzzyMinPixels: fuzzy_pixels.min,
+                          fuzzyMaxPixels: fuzzy_pixels.max,
+                          url1: testURI,
                           url2: null,
                           chaosMode: chaosMode }, aFilter);
         } else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL || items[0] == TYPE_PRINT) {
             if (items.length != 3)
                 throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to " + items[0];
 
             if (items[0] == TYPE_REFTEST_NOTEQUAL &&
                 expected_status == EXPECTED_FUZZY &&
                 (fuzzy_delta.min > 0 || fuzzy_pixels.min > 0)) {
                 throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": minimum fuzz must be zero for tests of type " + items[0];
             }
 
+            var [testURI, refURI] = runHttp
+                                  ? ServeFiles(principal, httpDepth,
+                                               listURL, [items[1], items[2]])
+                                  : [g.ioService.newURI(items[1], null, listURL),
+                                     g.ioService.newURI(items[2], null, listURL)];
+            var prettyPath = runHttp
+                           ? g.ioService.newURI(items[1], null, listURL).spec
+                           : testURI.spec;
+            secMan.checkLoadURIWithPrincipal(principal, testURI,
+                                             CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
+            secMan.checkLoadURIWithPrincipal(principal, refURI,
+                                             CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
             var type = items[0];
             if (g.compareStyloToGecko || g.compareRetainedDisplayLists) {
                 type = TYPE_REFTEST_EQUAL;
+                refURI = testURI;
 
                 // We expect twice as many assertion failures when running in
                 // styloVsGecko mode because we run each test twice: once in
                 // Stylo mode and once in Gecko mode.
                 minAsserts *= 2;
                 maxAsserts *= 2;
 
                 // Skip the test if it is expected to fail in both Stylo and
@@ -347,32 +397,30 @@ function ReadManifest(aURL, aFilter)
                 if (expected_status === EXPECTED_FAIL ||
                     expected_status === EXPECTED_RANDOM) {
                     expected_status = EXPECTED_DEATH;
                 }
             }
 
             AddTestItem({ type: type,
                           expected: expected_status,
-                          manifest: aURL.spec,
                           allowSilentFail: allow_silent_fail,
+                          prettyPath: prettyPath,
                           minAsserts: minAsserts,
                           maxAsserts: maxAsserts,
                           needsFocus: needs_focus,
                           slow: slow,
                           prefSettings1: testPrefSettings,
                           prefSettings2: refPrefSettings,
                           fuzzyMinDelta: fuzzy_delta.min,
                           fuzzyMaxDelta: fuzzy_delta.max,
                           fuzzyMinPixels: fuzzy_pixels.min,
                           fuzzyMaxPixels: fuzzy_pixels.max,
-                          runHttp: runHttp,
-                          httpDepth: httpDepth,
-                          url1: items[1],
-                          url2: items[2],
+                          url1: testURI,
+                          url2: refURI,
                           chaosMode: chaosMode }, aFilter);
         } else {
             throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": unknown test type " + items[0];
         }
     }
 }
 
 // Read all available data from an input stream and return it
@@ -639,17 +687,17 @@ function ExtractRange(matches, startInde
         };
     }
     return {
         min: Number(matches[startIndex]),
         max: Number(matches[startIndex + 1].substring(1))
     };
 }
 
-function ServeTestBase(aURL, depth) {
+function ServeFiles(manifestPrincipal, depth, aURL, files) {
     var listURL = aURL.QueryInterface(CI.nsIFileURL);
     var directory = listURL.file.parent;
 
     // Allow serving a tree that's an ancestor of the directory containing
     // the files so that they can use resources in ../ (etc.).
     var dirPath = "/";
     while (depth > 0) {
         dirPath = "/" + directory.leafName + dirPath;
@@ -664,67 +712,50 @@ function ServeTestBase(aURL, depth) {
     var secMan = CC[NS_SCRIPTSECURITYMANAGER_CONTRACTID]
                      .getService(CI.nsIScriptSecurityManager);
 
     var testbase = g.ioService.newURI("http://localhost:" + g.httpServerPort +
                                      path + dirPath);
 
     // Give the testbase URI access to XUL and XBL
     Services.perms.add(testbase, "allowXULXBL", Services.perms.ALLOW_ACTION);
-    return testbase;
-}
-
-function CreateUrls(test) {
-    let secMan = CC[NS_SCRIPTSECURITYMANAGER_CONTRACTID]
-                    .getService(CI.nsIScriptSecurityManager);
-
-    let manifestURL = g.ioService.newURI(test.manifest);
-    let principal = secMan.createCodebasePrincipal(manifestURL, {});
-
-    let testbase = manifestURL;
-    if (test.runHttp)
-        testbase = ServeTestBase(manifestURL, test.httpDepth)
 
     function FileToURI(file)
     {
-        if (file === null)
-            return file;
+        // Only serve relative URIs via the HTTP server, not absolute
+        // ones like about:blank.
+        var testURI = g.ioService.newURI(file, null, testbase);
 
-        var testURI = g.ioService.newURI(file, null, testbase);
-        secMan.checkLoadURIWithPrincipal(principal, testURI,
+        // XXX necessary?  manifestURL guaranteed to be file, others always HTTP
+        secMan.checkLoadURIWithPrincipal(manifestPrincipal, testURI,
                                          CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
+
         return testURI;
     }
 
-    let files = [test.url1, test.url2];
-    [test.url1, test.url2] = files.map(FileToURI);
-    if (test.url2 && g.compareStyloToGecko)
-        test.url2 = test.url1;
-
-    return test;
+    return files.map(FileToURI);
 }
 
 function AddTestItem(aTest, aFilter) {
     if (!aFilter)
         aFilter = [null, [], false];
 
-    var {url1, url2} = CreateUrls(Object.assign({}, aTest));
-
     var globalFilter = aFilter[0];
     var manifestFilter = aFilter[1];
     var invertManifest = aFilter[2];
-    if ((globalFilter && !globalFilter.test(url1.spec)) ||
+    if ((globalFilter && !globalFilter.test(aTest.url1.spec)) ||
         (manifestFilter &&
-         !(invertManifest ^ manifestFilter.test(url1.spec))))
+         !(invertManifest ^ manifestFilter.test(aTest.url1.spec))))
         return;
     if (g.focusFilterMode == FOCUS_FILTER_NEEDS_FOCUS_TESTS &&
         !aTest.needsFocus)
         return;
     if (g.focusFilterMode == FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS &&
         aTest.needsFocus)
         return;
 
-    if (url2 !== null)
-        aTest.identifier = [url1.spec, aTest.type, url2.spec];
+    if (aTest.url2 !== null)
+        aTest.identifier = [aTest.prettyPath, aTest.type, aTest.url2.spec];
     else
-        aTest.identifier = url1.spec;
+        aTest.identifier = aTest.prettyPath;
+
     g.urls.push(aTest);
 }
--- a/layout/tools/reftest/output.py
+++ b/layout/tools/reftest/output.py
@@ -1,15 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import threading
-from collections import defaultdict
 
 from mozlog.formatters import TbplFormatter
 from mozrunner.utils import get_stack_fixer_function
 
 
 class ReftestFormatter(TbplFormatter):
     """
     Formatter designed to preserve the legacy "tbpl" format in reftest.
@@ -125,36 +124,31 @@ class OutputHandler(object):
     raw data logged from reftest.js to an appropriate structured log action,
     where applicable.
     """
 
     def __init__(self, log, utilityPath, symbolsPath=None):
         self.stack_fixer_function = get_stack_fixer_function(utilityPath, symbolsPath)
         self.log = log
         self.proc_name = None
-        self.results = defaultdict(int)
 
     def __call__(self, line):
         # need to return processed messages to appease remoteautomation.py
         if not line.strip():
             return []
         line = line.decode('utf-8', errors='replace')
 
         try:
             data = json.loads(line)
         except ValueError:
             self.verbatim(line)
             return [line]
 
         if isinstance(data, dict) and 'action' in data:
-            if data['action'] == 'results':
-                for k, v in data['results'].items():
-                    self.results[k] += v
-            else:
-                self.log.log_raw(data)
+            self.log.log_raw(data)
         else:
             self.verbatim(json.dumps(data))
 
         return [data]
 
     def verbatim(self, line):
         if self.stack_fixer_function:
             line = self.stack_fixer_function(line)
--- a/layout/tools/reftest/reftest.jsm
+++ b/layout/tools/reftest/reftest.jsm
@@ -316,20 +316,20 @@ function InitAndStartRefTests()
         //g.browser.loadURI('data:text/plain,' + ex);
         ++g.testResults.Exception;
         logger.error("EXCEPTION: " + ex);
         DoneTests();
     }
 
     // Focus the content browser.
     if (g.focusFilterMode != FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS) {
-        g.browser.addEventListener("focus", ReadTests, true);
+        g.browser.addEventListener("focus", StartTests, true);
         g.browser.focus();
     } else {
-        ReadTests();
+        StartTests();
     }
 }
 
 function StartHTTPServer()
 {
     g.server.registerContentType("sjs", "sjs");
     g.server.start(-1);
     g.httpServerPort = g.server.identity.primaryPort;
@@ -341,97 +341,23 @@ function Shuffle(array)
     for (var i = array.length - 1; i > 0; i--) {
         var j = Math.floor(Math.random() * (i + 1));
         var temp = array[i];
         array[i] = array[j];
         array[j] = temp;
     }
 }
 
-function ReadTests() {
-    try {
-        if (g.focusFilterMode != FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS) {
-            g.browser.removeEventListener("focus", ReadTests, true);
-        }
-
-        g.urls = [];
-        var prefs = Components.classes["@mozilla.org/preferences-service;1"].
-                    getService(Components.interfaces.nsIPrefBranch);
-
-        /* There are three modes implemented here:
-         * 1) reftest.manifests
-         * 2) reftest.manifests and reftest.manifests.dumpTests
-         * 3) reftest.tests
-         *
-         * The first will parse the specified manifests, then immediately
-         * run the tests. The second will parse the manifests, save the test
-         * objects to a file and exit. The third will load a file of test
-         * objects and run them.
-         *
-         * The latter two modes are used to pass test data back and forth
-         * with python harness.
-        */
-        let manifests = prefs.getCharPref("reftest.manifests", null);
-        let dumpTests = prefs.getCharPref("reftest.manifests.dumpTests", null);
-        let testList = prefs.getCharPref("reftest.tests", null);
-
-        if ((testList && manifests) || !(testList || manifests)) {
-            logger.error("Exactly one of reftest.manifests or reftest.tests must be specified.");
-            DoneTests();
-        }
-
-        if (testList) {
-            let promise = OS.File.read(testList).then(function onSuccess(array) {
-                let decoder = new TextDecoder();
-                g.urls = JSON.parse(decoder.decode(array)).map(CreateUrls);
-                StartTests();
-            });
-        } else if (manifests) {
-            // Parse reftest manifests
-            manifests = JSON.parse(manifests);
-            g.urlsFilterRegex = manifests[null];
-
-            var globalFilter = manifests.hasOwnProperty("") ? new RegExp(manifests[""]) : null;
-            var manifestURLs = Object.keys(manifests);
-
-            // Ensure we read manifests from higher up the directory tree first so that we
-            // process includes before reading the included manifest again
-            manifestURLs.sort(function(a,b) {return a.length - b.length})
-            manifestURLs.forEach(function(manifestURL) {
-                logger.info("Reading manifest " + manifestURL);
-                var filter = manifests[manifestURL] ? new RegExp(manifests[manifestURL]) : null;
-                ReadTopManifest(manifestURL, [globalFilter, filter, false]);
-            });
-
-            if (dumpTests) {
-                let encoder = new TextEncoder();
-                let tests = encoder.encode(JSON.stringify(g.urls));
-                OS.File.writeAtomic(dumpTests, tests, {flush: true}).then(
-                  function onSuccess() {
-                    DoneTests();
-                  },
-                  function onFailure(reason) {
-                    logger.error("failed to write test data: " + reason);
-                    DoneTests();
-                  }
-                )
-            } else {
-                g.manageSuite = true;
-                g.urls = g.urls.map(CreateUrls);
-                StartTests();
-            }
-        }
-    } catch(e) {
-        ++g.testResults.Exception;
-        logger.error("EXCEPTION: " + e);
-    }
-}
-
 function StartTests()
 {
+    if (g.focusFilterMode != FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS) {
+        g.browser.removeEventListener("focus", StartTests, true);
+    }
+
+    var manifests;
     /* These prefs are optional, so we don't need to spit an error to the log */
     try {
         var prefs = Components.classes["@mozilla.org/preferences-service;1"].
                     getService(Components.interfaces.nsIPrefBranch);
     } catch(e) {
         logger.error("EXCEPTION: " + e);
     }
 
@@ -457,17 +383,38 @@ function StartTests()
     }
 
     g.runSlowTests = prefs.getIntPref("reftest.skipslowtests", false);
 
     if (g.shuffle) {
         g.noCanvasCache = true;
     }
 
+    g.urls = [];
+
     try {
+        var manifests = JSON.parse(prefs.getCharPref("reftest.manifests"));
+        g.urlsFilterRegex = manifests[null];
+    } catch(e) {
+        logger.error("Unable to find reftest.manifests pref.  Please ensure your profile is setup properly");
+        DoneTests();
+    }
+
+    try {
+        var globalFilter = manifests.hasOwnProperty("") ? new RegExp(manifests[""]) : null;
+        var manifestURLs = Object.keys(manifests);
+
+        // Ensure we read manifests from higher up the directory tree first so that we
+        // process includes before reading the included manifest again
+        manifestURLs.sort(function(a,b) {return a.length - b.length})
+        manifestURLs.forEach(function(manifestURL) {
+            logger.info("Reading manifest " + manifestURL);
+            var filter = manifests[manifestURL] ? new RegExp(manifests[manifestURL]) : null;
+            ReadTopManifest(manifestURL, [globalFilter, filter, false]);
+        });
         BuildUseCounts();
 
         // Filter tests which will be skipped to get a more even distribution when chunking
         // tURLs is a temporary array containing all active tests
         var tURLs = new Array();
         for (var i = 0; i < g.urls.length; ++i) {
             if (g.urls[i].expected == EXPECTED_DEATH)
                 continue;
@@ -497,17 +444,17 @@ function StartTests()
             end = g.thisChunk == g.totalChunks ? g.urls.length : g.urls.indexOf(tURLs[end + 1]) - 1;
 
             logger.info("Running chunk " + g.thisChunk + " out of " + g.totalChunks + " chunks.  " +
                 "tests " + (start+1) + "-" + end + "/" + g.urls.length);
 
             g.urls = g.urls.slice(start, end);
         }
 
-        if (g.manageSuite && g.startAfter === undefined && !g.suiteStarted) {
+        if (g.startAfter === undefined && !g.suiteStarted) {
             var ids = g.urls.map(function(obj) {
                 return obj.identifier;
             });
             var suite = prefs.getCharPref('reftest.suite', 'reftest');
             logger.suiteStart(ids, suite, {"skipped": g.urls.length - numActiveTests});
             g.suiteStarted = true
         }
 
@@ -773,22 +720,18 @@ function StartCurrentURI(aURLTargetType)
         } else {
             SendLoadTest(type, g.currentURL, g.currentURLTargetType, g.loadTimeout);
         }
     }
 }
 
 function DoneTests()
 {
-    if (g.manageSuite) {
-        g.suiteStarted = false
-        logger.suiteEnd({'results': g.testResults});
-    } else {
-        logger._logData('results', {results: g.testResults});
-    }
+    logger.suiteEnd({'results': g.testResults});
+    g.suiteStarted = false
     logger.info("Slowest test took " + g.slowestTestTime + "ms (" + g.slowestTestURL + ")");
     logger.info("Total canvas count = " + g.recycledCanvases.length);
     if (g.failedUseWidgetLayers) {
         LogWidgetLayersFailure();
     }
 
     function onStopped() {
         let appStartup = CC["@mozilla.org/toolkit/app-startup;1"].getService(CI.nsIAppStartup);
--- a/layout/tools/reftest/remotereftest.py
+++ b/layout/tools/reftest/remotereftest.py
@@ -1,22 +1,22 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from contextlib import closing
+import sys
 import logging
 import os
 import psutil
 import signal
-import sys
+import time
 import tempfile
-import time
 import traceback
 import urllib2
-from contextlib import closing
 
 import mozdevice
 import mozinfo
 from automation import Automation
 from remoteautomation import RemoteAutomation, fennecLogcatFilters
 
 from output import OutputHandler
 from runreftest import RefTest, ReftestResolver
@@ -138,17 +138,16 @@ class ReftestServer:
                 self.automation.log.info("Failed to shutdown server at %s" %
                                          self.shutdownURL)
                 traceback.print_exc()
                 self._process.kill()
 
 
 class RemoteReftest(RefTest):
     use_marionette = False
-    parse_manifest = False
     remoteApp = ''
     resolver_cls = RemoteReftestResolver
 
     def __init__(self, automation, devicemanager, options, scriptDir):
         RefTest.__init__(self)
         self.automation = automation
         self._devicemanager = devicemanager
         self.scriptDir = scriptDir
@@ -163,21 +162,21 @@ class RemoteReftest(RefTest):
             self.SERVER_STARTUP_TIMEOUT = 180
         else:
             self.SERVER_STARTUP_TIMEOUT = 90
         self.automation.deleteANRs()
         self.automation.deleteTombstones()
         self._devicemanager.removeDir(self.remoteCache)
 
         self._populate_logger(options)
-        self.outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath)
+        outputHandler = OutputHandler(self.log, options.utilityPath, options.symbolsPath)
         # RemoteAutomation.py's 'messageLogger' is also used by mochitest. Mimic a mochitest
         # MessageLogger object to re-use this code path.
-        self.outputHandler.write = self.outputHandler.__call__
-        self.automation._processArgs['messageLogger'] = self.outputHandler
+        outputHandler.write = outputHandler.__call__
+        self.automation._processArgs['messageLogger'] = outputHandler
 
     def findPath(self, paths, filename=None):
         for path in paths:
             p = path
             if filename:
                 p = os.path.join(p, filename)
             if os.path.exists(self.getFullPath(p)):
                 return path
@@ -255,22 +254,22 @@ class RemoteReftest(RefTest):
                         except Exception as e:
                             self.log.info("Failed to kill process %d: %s" % (proc.pid, str(e)))
                     else:
                         self.log.info("NOT killing %s (not an orphan?)" % procd)
             except Exception:
                 # may not be able to access process info for all processes
                 continue
 
-    def createReftestProfile(self, options, startAfter=None, **kwargs):
+    def createReftestProfile(self, options, manifest, startAfter=None):
         profile = RefTest.createReftestProfile(self,
                                                options,
+                                               manifest,
                                                server=options.remoteWebServer,
-                                               port=options.httpPort,
-                                               **kwargs)
+                                               port=options.httpPort)
         if startAfter is not None:
             print ("WARNING: Continuing after a crash is not supported for remote "
                    "reftest yet.")
         profileDir = profile.profile
 
         prefs = {}
         prefs["app.update.url.android"] = ""
         prefs["browser.firstrun.show.localepicker"] = False
@@ -279,21 +278,16 @@ class RemoteReftest(RefTest):
         # move necko cache to a location that can be cleaned up
         prefs["browser.cache.disk.parent_directory"] = self.remoteCache
 
         prefs["layout.css.devPixelsPerPx"] = "1.0"
         # Because Fennec is a little wacky (see bug 1156817) we need to load the
         # reftest pages at 1.0 zoom, rather than zooming to fit the CSS viewport.
         prefs["apz.allow_zooming"] = False
 
-        if options.totalChunks:
-            prefs['reftest.totalChunks'] = options.totalChunks
-        if options.thisChunk:
-            prefs['reftest.thisChunk'] = options.thisChunk
-
         # Set the extra prefs.
         profile.set_preferences(prefs)
 
         try:
             self._devicemanager.pushDir(profileDir, options.remoteProfile)
             self._devicemanager.chmodDir(options.remoteProfile)
         except mozdevice.DMError:
             print "Automation Error: Failed to copy profiledir to device"
@@ -334,46 +328,33 @@ class RemoteReftest(RefTest):
 
     def buildBrowserEnv(self, options, profileDir):
         browserEnv = RefTest.buildBrowserEnv(self, options, profileDir)
         # remove desktop environment not used on device
         if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
             del browserEnv["XPCOM_MEM_BLOAT_LOG"]
         return browserEnv
 
-    def runApp(self, options, cmdargs=None, timeout=None, debuggerInfo=None, symbolsPath=None,
-               valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None, **profileArgs):
-        if cmdargs is None:
-            cmdargs = []
-
-        if self.use_marionette:
-            cmdargs.append('-marionette')
-
-        binary = options.app
-        profile = self.createReftestProfile(options, **profileArgs)
-
-        # browser environment
-        env = self.buildBrowserEnv(options, profile.profile)
-
-        self.log.info("Running with e10s: {}".format(options.e10s))
+    def runApp(self, profile, binary, cmdargs, env,
+               timeout=None, debuggerInfo=None,
+               symbolsPath=None, options=None,
+               valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None):
         status, lastTestSeen = self.automation.runApp(None, env,
                                                       binary,
                                                       profile.profile,
                                                       cmdargs,
                                                       utilityPath=options.utilityPath,
                                                       xrePath=options.xrePath,
                                                       debuggerInfo=debuggerInfo,
                                                       symbolsPath=symbolsPath,
                                                       timeout=timeout)
         if status == 1:
             # when max run time exceeded, avoid restart
             lastTestSeen = RefTest.TEST_SEEN_FINAL
-
-        self.cleanup(profile.profile)
-        return status, lastTestSeen, self.outputHandler.results
+        return status, lastTestSeen
 
     def cleanup(self, profileDir):
         # Pull results back from device
         if self.remoteLogFile and \
                 self._devicemanager.fileExists(self.remoteLogFile):
             self._devicemanager.getFile(self.remoteLogFile, self.localLogName)
         else:
             print "WARNING: Unable to retrieve log file (%s) from remote " \
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -13,35 +13,32 @@ import json
 import multiprocessing
 import os
 import platform
 import re
 import shutil
 import signal
 import subprocess
 import sys
-import tempfile
 import threading
 from datetime import datetime, timedelta
 
 SCRIPT_DIRECTORY = os.path.abspath(
     os.path.realpath(os.path.dirname(__file__)))
 if SCRIPT_DIRECTORY not in sys.path:
     sys.path.insert(0, SCRIPT_DIRECTORY)
 
 import mozcrash
 import mozdebug
-import mozfile
 import mozinfo
 import mozleak
 import mozlog
 import mozprocess
 import mozprofile
 import mozrunner
-from manifestparser import TestManifest, filters as mpf
 from mozrunner.utils import get_stack_fixer_function, test_environment
 from mozscreenshot import printstatus, dump_screen
 
 try:
     from marionette_driver.addons import Addons
     from marionette_harness import Marionette
 except ImportError, e:
     # Defer ImportError until attempt to use Marionette
@@ -224,28 +221,26 @@ class ReftestResolver(object):
             else:
                 manifests[key] = "|".join(list(manifests[key]))
         return manifests
 
 
 class RefTest(object):
     TEST_SEEN_INITIAL = 'reftest'
     TEST_SEEN_FINAL = 'Main app process exited normally'
+    use_marionette = True
     oldcwd = os.getcwd()
-    parse_manifest = True
     resolver_cls = ReftestResolver
-    use_marionette = True
 
     def __init__(self):
         update_mozinfo()
         self.lastTestSeen = self.TEST_SEEN_INITIAL
         self.haveDumpedScreen = False
         self.resolver = self.resolver_cls()
         self.log = None
-        self.testDumpFile = os.path.join(tempfile.gettempdir(), 'reftests.json')
 
     def _populate_logger(self, options):
         if self.log:
             return
 
         self.log = getattr(options, 'log', None)
         if self.log:
             return
@@ -259,56 +254,57 @@ class RefTest(object):
             options.log_tbpl_level = fmt_options['level'] = 'debug'
         self.log = mozlog.commandline.setup_logging(
             "reftest harness", options, {"tbpl": sys.stdout}, fmt_options)
 
     def getFullPath(self, path):
         "Get an absolute path relative to self.oldcwd."
         return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
 
-    def createReftestProfile(self, options, tests=None, manifests=None,
-                             server='localhost', port=0, profile_to_clone=None,
-                             startAfter=None, prefs=None):
+    def createReftestProfile(self, options, manifests, server='localhost', port=0,
+                             profile_to_clone=None, startAfter=None):
         """Sets up a profile for reftest.
 
         :param options: Object containing command line options
-        :param tests: List of test objects to run
-        :param manifests: List of manifest files to parse (only takes effect
-                          if tests were not passed in)
+        :param manifests: Dictionary of the form {manifest_path: [filters]}
         :param server: Server name to use for http tests
         :param profile_to_clone: Path to a profile to use as the basis for the
                                  test profile
-        :param startAfter: Start running tests after the specified test id
-        :param prefs: Extra preferences to set in the profile
         """
+
         locations = mozprofile.permissions.ServerLocations()
         locations.add_host(server, scheme='http', port=port)
         locations.add_host(server, scheme='https', port=port)
 
         # Set preferences for communication between our command line arguments
         # and the reftest harness.  Preferences that are required for reftest
         # to work should instead be set in reftest-preferences.js .
-        prefs = prefs or {}
+        prefs = {}
         prefs['reftest.timeout'] = options.timeout * 1000
+        if options.totalChunks:
+            prefs['reftest.totalChunks'] = options.totalChunks
+        if options.thisChunk:
+            prefs['reftest.thisChunk'] = options.thisChunk
         if options.logFile:
             prefs['reftest.logFile'] = options.logFile
         if options.ignoreWindowSize:
             prefs['reftest.ignoreWindowSize'] = True
         if options.shuffle:
             prefs['reftest.shuffle'] = True
         if options.repeat:
             prefs['reftest.repeat'] = options.repeat
         if options.runUntilFailure:
             prefs['reftest.runUntilFailure'] = True
         if options.verify:
             prefs['reftest.verify'] = True
         if options.cleanupCrashes:
             prefs['reftest.cleanupPendingCrashes'] = True
         prefs['reftest.focusFilterMode'] = options.focusFilterMode
         prefs['reftest.logLevel'] = options.log_tbpl_level or 'info'
+        prefs['reftest.manifests'] = json.dumps(manifests)
         prefs['reftest.suite'] = options.suite
 
         if startAfter not in (None, self.TEST_SEEN_INITIAL, self.TEST_SEEN_FINAL):
             self.log.info("Setting reftest.startAfter to %s" % startAfter)
             prefs['reftest.startAfter'] = startAfter
 
         # Unconditionally update the e10s pref.
         if options.e10s:
@@ -379,24 +375,16 @@ class RefTest(object):
                   'preferences': prefs,
                   'locations': locations,
                   'whitelistpaths': sandbox_whitelist_paths}
         if profile_to_clone:
             profile = mozprofile.Profile.clone(profile_to_clone, **kwargs)
         else:
             profile = mozprofile.Profile(**kwargs)
 
-        if tests:
-            testlist = os.path.join(profile.profile, 'reftests.json')
-            with open(testlist, 'w') as fh:
-                json.dump(tests, fh)
-            profile.set_preferences({'reftest.tests': testlist})
-        elif manifests:
-            profile.set_preferences({'reftest.manifests': json.dumps(manifests)})
-
         if os.path.join(here, 'chrome') not in options.extraProfileFiles:
             options.extraProfileFiles.append(os.path.join(here, 'chrome'))
 
         self.copyExtraFilesToProfile(options, profile)
         return profile
 
     def environment(self, **kwargs):
         kwargs['log'] = self.log
@@ -666,33 +654,20 @@ class RefTest(object):
                     process.kill(sig=signal.SIGABRT)
                 except OSError:
                     # https://bugzilla.mozilla.org/show_bug.cgi?id=921509
                     self.log.info("Can't trigger Breakpad, process no longer exists")
                 return
         self.log.info("Can't trigger Breakpad, just killing process")
         process.kill()
 
-    def runApp(self, options, cmdargs=None, timeout=None, debuggerInfo=None,
-               symbolsPath=None, valgrindPath=None, valgrindArgs=None,
-               valgrindSuppFiles=None, **profileArgs):
-
-        if cmdargs is None:
-            cmdargs = []
-
-        if self.use_marionette:
-            cmdargs.append('-marionette')
-
-        binary = options.app
-        profile = self.createReftestProfile(options, **profileArgs)
-
-        # browser environment
-        env = self.buildBrowserEnv(options, profile.profile)
-
-        self.log.info("Running with e10s: {}".format(options.e10s))
+    def runApp(self, profile, binary, cmdargs, env,
+               timeout=None, debuggerInfo=None,
+               symbolsPath=None, options=None,
+               valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None):
 
         def timeoutHandler():
             self.handleTimeout(
                 timeout, proc, options.utilityPath, debuggerInfo)
 
         interactive = False
         debug_args = None
         if debuggerInfo:
@@ -789,122 +764,98 @@ class RefTest(object):
             self.lastTestSeen = self.TEST_SEEN_FINAL
 
         crashed = mozcrash.log_crashes(self.log, os.path.join(profile.profile, 'minidumps'),
                                        symbolsPath, test=self.lastTestSeen)
         if not status and crashed:
             status = 1
 
         runner.cleanup()
-        self.cleanup(profile.profile)
 
         if marionette_exception is not None:
             exc, value, tb = marionette_exception
             raise exc, value, tb
 
-        self.log.info("Process mode: {}".format('e10s' if options.e10s else 'non-e10s'))
-        return status, self.lastTestSeen, outputHandler.results
-
-    def getActiveTests(self, manifests, options, testDumpFile=None):
-        # These prefs will cause reftest.jsm to parse the manifests,
-        # dump the resulting tests to a file, and exit.
-        prefs = {
-            'reftest.manifests': json.dumps(manifests),
-            'reftest.manifests.dumpTests': testDumpFile or self.testDumpFile,
-        }
-        cmdargs = []  # ['-headless']
-        status, _, _ = self.runApp(options, cmdargs=cmdargs, prefs=prefs)
-
-        with open(self.testDumpFile, 'r') as fh:
-            tests = json.load(fh)
-
-        if os.path.isfile(self.testDumpFile):
-            mozfile.remove(self.testDumpFile)
-
-        for test in tests:
-            # Name and path are expected by manifestparser, but not used in reftest.
-            test['name'] = test['path'] = test['url1']
-
-        mp = TestManifest(strict=False)
-        mp.tests = tests
-
-        filters = []
-        if options.totalChunks:
-            filters.append(mpf.chunk_by_slice(options.thisChunk, options.totalChunks))
-
-        tests = mp.active_tests(exists=False, filters=filters)
-        return tests
+        return status, self.lastTestSeen
 
     def runSerialTests(self, manifests, options, cmdargs=None):
         debuggerInfo = None
         if options.debugger:
             debuggerInfo = mozdebug.get_debugger_info(options.debugger, options.debuggerArgs,
                                                       options.debuggerInteractive)
 
-        tests = None
-        if self.parse_manifest:
-            tests = self.getActiveTests(manifests, options)
-
-            ids = [t['identifier'] for t in tests]
-            self.log.suite_start(ids, name=options.suite)
-
+        profileDir = None
         startAfter = None  # When the previous run crashed, we skip the tests we ran before
         prevStartAfter = None
         for i in itertools.count():
-            status, startAfter, results = self.runApp(
-                options,
-                tests=tests,
-                manifests=manifests,
-                cmdargs=cmdargs,
-                # We generally want the JS harness or marionette
-                # to handle timeouts if they can.
-                # The default JS harness timeout is currently
-                # 300 seconds (default options.timeout).
-                # The default Marionette socket timeout is
-                # currently 360 seconds.
-                # Give the JS harness extra time to deal with
-                # its own timeouts and try to usually exceed
-                # the 360 second marionette socket timeout.
-                # See bug 479518 and bug 1414063.
-                timeout=options.timeout + 70.0,
-                symbolsPath=options.symbolsPath,
-                debuggerInfo=debuggerInfo
-            )
-            mozleak.process_leak_log(self.leakLogFile,
-                                     leak_thresholds=options.leakThresholds,
-                                     stack_fixer=get_stack_fixer_function(options.utilityPath,
-                                                                          options.symbolsPath))
+            try:
+                if cmdargs is None:
+                    cmdargs = []
+
+                if self.use_marionette:
+                    cmdargs.append('-marionette')
+
+                profile = self.createReftestProfile(options,
+                                                    manifests,
+                                                    startAfter=startAfter)
+                profileDir = profile.profile  # name makes more sense
+
+                # browser environment
+                browserEnv = self.buildBrowserEnv(options, profileDir)
 
-            if status == 0:
-                break
-
-            if startAfter == self.TEST_SEEN_FINAL:
-                self.log.info("Finished running all tests, skipping resume "
-                              "despite non-zero status code: %s" % status)
-                break
+                self.log.info("Running with e10s: {}".format(options.e10s))
+                status, startAfter = self.runApp(profile,
+                                                 binary=options.app,
+                                                 cmdargs=cmdargs,
+                                                 env=browserEnv,
+                                                 # We generally want the JS harness or marionette
+                                                 # to handle timeouts if they can.
+                                                 # The default JS harness timeout is currently
+                                                 # 300 seconds (default options.timeout).
+                                                 # The default Marionette socket timeout is
+                                                 # currently 360 seconds.
+                                                 # Give the JS harness extra time to deal with
+                                                 # its own timeouts and try to usually exceed
+                                                 # the 360 second marionette socket timeout.
+                                                 # See bug 479518 and bug 1414063.
+                                                 timeout=options.timeout + 70.0,
+                                                 symbolsPath=options.symbolsPath,
+                                                 options=options,
+                                                 debuggerInfo=debuggerInfo)
+                self.log.info("Process mode: {}".format('e10s' if options.e10s else 'non-e10s'))
+                mozleak.process_leak_log(self.leakLogFile,
+                                         leak_thresholds=options.leakThresholds,
+                                         stack_fixer=get_stack_fixer_function(options.utilityPath,
+                                                                              options.symbolsPath))
+                if status == 0:
+                    break
 
-            if startAfter is not None and options.shuffle:
-                self.log.error("Can not resume from a crash with --shuffle "
-                               "enabled. Please consider disabling --shuffle")
-                break
-            if startAfter is not None and options.maxRetries <= i:
-                self.log.error("Hit maximum number of allowed retries ({}) "
-                               "in the test run".format(options.maxRetries))
-                break
-            if startAfter == prevStartAfter:
-                # If the test stuck on the same test, or there the crashed
-                # test appeared more then once, stop
-                self.log.error("Force stop because we keep running into "
-                               "test \"{}\"".format(startAfter))
-                break
-            prevStartAfter = startAfter
-            # TODO: we need to emit an SUITE-END log if it crashed
+                if startAfter == self.TEST_SEEN_FINAL:
+                    self.log.info("Finished running all tests, skipping resume "
+                                  "despite non-zero status code: %s" % status)
+                    break
 
-        if self.parse_manifest:
-            self.log.suite_end(extra={'results': results})
+                if startAfter is not None and options.shuffle:
+                    self.log.error("Can not resume from a crash with --shuffle "
+                                   "enabled. Please consider disabling --shuffle")
+                    break
+                if startAfter is not None and options.maxRetries <= i:
+                    self.log.error("Hit maximum number of allowed retries ({}) "
+                                   "in the test run".format(options.maxRetries))
+                    break
+                if startAfter == prevStartAfter:
+                    # If the test stuck on the same test, or there the crashed
+                    # test appeared more then once, stop
+                    self.log.error("Force stop because we keep running into "
+                                   "test \"{}\"".format(startAfter))
+                    break
+                prevStartAfter = startAfter
+                # TODO: we need to emit an SUITE-END log if it crashed
+            finally:
+                self.cleanup(profileDir)
         return status
 
     def copyExtraFilesToProfile(self, options, profile):
         "Copy extra files or dirs specified on the command line to the testing profile."
         profileDir = profile.profile
         for f in options.extraProfileFiles:
             abspath = self.getFullPath(f)
             if os.path.isfile(abspath):
--- a/testing/mozbase/mozlog/mozlog/structuredlog.py
+++ b/testing/mozbase/mozlog/mozlog/structuredlog.py
@@ -299,17 +299,17 @@ class StructuredLogger(object):
         self._log_data("suite_start", data)
 
     @log_action(Dict(Any, "extra", default=None, optional=True))
     def suite_end(self, data):
         """Log a suite_end message"""
         if not self._ensure_suite_state('suite_end', data):
             return
 
-        self._log_data("suite_end", data)
+        self._log_data("suite_end")
 
     @log_action(TestId("test"),
                 Unicode("path", default=None, optional=True))
     def test_start(self, data):
         """Log a test_start message
 
         :param test: Identifier of the test that will run.
         :param path: Path to test relative to some base (typically the root of