Bug 998206. Add --shuffle option to runreftests.py and 'mach reftest'. r=dbaron
authorRobert O'Callahan <robert@ocallahan.org>
Mon, 03 Mar 2014 18:13:01 +1300
changeset 180323 a344959a575677d2c6a92d57d2787489bffe74bb
parent 180322 776f4e1090c83c7fe332c929683c0be47b7457a4
child 180324 f3aaa298ed2ba84a8dccf8570c8f729ef81ae90a
push id272
push userpvanderbeken@mozilla.com
push dateMon, 05 May 2014 16:31:18 +0000
reviewersdbaron
bugs998206
milestone31.0a1
Bug 998206. Add --shuffle option to runreftests.py and 'mach reftest'. r=dbaron
layout/tools/reftest/mach_commands.py
layout/tools/reftest/reftest.js
layout/tools/reftest/runreftest.py
--- a/layout/tools/reftest/mach_commands.py
+++ b/layout/tools/reftest/mach_commands.py
@@ -201,34 +201,36 @@ class ReftestRunner(MozbuildObject):
         options.b2gPath = b2g_home
         options.logcat_dir = self.reftest_dir
         options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
         options.xrePath = xre_path
         options.ignoreWindowSize = True
         return reftest.run_remote_reftests(parser, options, args)
 
     def run_desktop_test(self, test_file=None, filter=None, suite=None,
-            debugger=None, parallel=False, e10s=False, this_chunk=None,
-            total_chunks=None):
+            debugger=None, parallel=False, shuffle=False,
+            e10s=False, this_chunk=None, total_chunks=None):
         """Runs a reftest.
 
         test_file is a path to a test file. It can be a relative path from the
         top source directory, an absolute filename, or a directory containing
         test files.
 
         filter is a regular expression (in JS syntax, as could be passed to the
         RegExp constructor) to select which reftests to run from the manifest.
 
         suite is the type of reftest to run. It can be one of ('reftest',
         'crashtest').
 
         debugger is the program name (in $PATH) or the full path of the
         debugger to run.
 
         parallel indicates whether tests should be run in parallel or not.
+
+        shuffle indicates whether to run tests in random order.
         """
 
         if suite not in ('reftest', 'reftest-ipc', 'crashtest', 'crashtest-ipc'):
             raise Exception('None or unrecognized reftest suite type.')
 
         env = {}
         extra_args = []
 
@@ -244,16 +246,19 @@ class ReftestRunner(MozbuildObject):
 
         if debugger:
             extra_args.append('--debugger=%s' % debugger)
             pass_thru = True
 
         if parallel:
             extra_args.append('--run-tests-in-parallel')
 
+        if shuffle:
+            extra_args.append('--shuffle')
+
         if e10s:
             extra_args.append('--e10s')
 
         if this_chunk:
             extra_args.append('--this-chunk=%s' % this_chunk)
 
         if total_chunks:
             extra_args.append('--total-chunks=%s' % total_chunks)
@@ -284,16 +289,20 @@ def ReftestCommand(func):
         help='Reftest manifest file, or a directory in which to select '
              'reftest.list. If omitted, the entire test suite is executed.')
     func = path(func)
 
     parallel = CommandArgument('--parallel', action='store_true',
         help='Run tests in parallel.')
     func = parallel(func)
 
+    shuffle = CommandArgument('--shuffle', action='store_true',
+        help='Run tests in random order.')
+    func = shuffle(func)
+
     e10s = CommandArgument('--e10s', action='store_true',
         help='Use content processes.')
     func = e10s(func)
 
     totalChunks = CommandArgument('--total-chunks',
         help = 'How many chunks to split the tests up into.')
     func = totalChunks(func)
 
--- a/layout/tools/reftest/reftest.js
+++ b/layout/tools/reftest/reftest.js
@@ -36,16 +36,17 @@ const NS_OBSERVER_SERVICE_CONTRACTID =
           "@mozilla.org/observer-service;1";
 
 Components.utils.import("resource://gre/modules/FileUtils.jsm");
 
 var gLoadTimeout = 0;
 var gTimeoutHook = null;
 var gRemote = false;
 var gIgnoreWindowSize = false;
+var gShuffle = false;
 var gTotalChunks = 0;
 var gThisChunk = 0;
 var gContainingWindow = null;
 var gURLFilterRegex = null;
 const FOCUS_FILTER_ALL_TESTS = "all";
 const FOCUS_FILTER_NEEDS_FOCUS_TESTS = "needs-focus";
 const FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS = "non-needs-focus";
 var gFocusFilterMode = FOCUS_FILTER_ALL_TESTS;
@@ -427,16 +428,27 @@ function InitAndStartRefTests()
 
 function StartHTTPServer()
 {
     gServer.registerContentType("sjs", "sjs");
     gServer.start(-1);
     gHttpServerPort = gServer.identity.primaryPort;
 }
 
+// Perform a Fisher-Yates shuffle of the array.
+function Shuffle(array)
+{
+    for (var i = array.length - 1; i > 0; i--) {
+        var j = Math.floor(Math.random() * (i + 1));
+        var temp = array[i];
+        array[i] = array[j];
+        array[j] = temp;
+    }
+}
+
 function StartTests()
 {
     var uri;
 #if BOOTSTRAP
     /* These prefs are optional, so we don't need to spit an error to the log */
     try {
         var prefs = Components.classes["@mozilla.org/preferences-service;1"].
                     getService(Components.interfaces.nsIPrefBranch);
@@ -446,16 +458,22 @@ function StartTests()
 
     try {
         gNoCanvasCache = prefs.getIntPref("reftest.nocache");
     } catch(e) {
         gNoCanvasCache = false;
     }
 
     try {
+      gShuffle = prefs.getBoolPref("reftest.shuffle");
+    } catch (e) {
+      gShuffle = false;
+    }
+
+    try {
         gRunSlowTests = prefs.getIntPref("reftest.skipslowtests");
     } catch(e) {
         gRunSlowTests = false;
     }
 
     try {
         uri = prefs.getCharPref("reftest.uri");
     } catch(e) {
@@ -479,16 +497,21 @@ function StartTests()
 
         uri = args.uri;
     } catch (e) {
         ++gTestResults.Exception;
         gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | EXCEPTION: " + ex + "\n");
         DoneTests();
     }
 #endif
+
+    if (gShuffle) {
+        gNoCanvasCache = true;
+    }
+
     try {
         ReadTopManifest(uri);
         BuildUseCounts();
 
         // Filter tests which will be skipped to get a more even distribution when chunking
         // tURLs is a temporary array containing all active tests
         var tURLs = new Array();
         for (var i = 0; i < gURLs.length; ++i) {
@@ -517,16 +540,21 @@ function StartTests()
             // gURLs array which prevents skipped tests from showing up in the log
             start = gThisChunk == 1 ? 0 : gURLs.indexOf(tURLs[start]);
             end = gThisChunk == gTotalChunks ? gURLs.length : gURLs.indexOf(tURLs[end + 1]) - 1;
             gURLs = gURLs.slice(start, end);
 
             gDumpLog("REFTEST INFO | Running chunk " + gThisChunk + " out of " + gTotalChunks + " chunks.  ");
             gDumpLog("tests " + (start+1) + "-" + end + "/" + gURLs.length + "\n");
         }
+
+        if (gShuffle) {
+            Shuffle(gURLs);
+        }
+
         gTotalTests = gURLs.length;
 
         if (!gTotalTests)
             throw "No tests to run";
 
         gURICanvases = {};
         StartCurrentTest();
     } catch (ex) {
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -156,16 +156,18 @@ class RefTest(object):
     if options.thisChunk:
       prefs['reftest.thisChunk'] = options.thisChunk
     if options.logFile:
       prefs['reftest.logFile'] = options.logFile
     if options.ignoreWindowSize:
       prefs['reftest.ignoreWindowSize'] = True
     if options.filter:
       prefs['reftest.filter'] = options.filter
+    if options.shuffle:
+      prefs['reftest.shuffle'] = True
     prefs['reftest.focusFilterMode'] = options.focusFilterMode
 
     if options.e10s:
       prefs['browser.tabs.remote.autostart'] = True
 
     for v in options.extraPrefs:
       thispref = v.split('=')
       if len(thispref) < 2:
@@ -436,16 +438,21 @@ class ReftestOptions(OptionParser):
 
     self.add_option("--filter",
                     action = "store", type="string", dest = "filter",
                     help = "specifies a regular expression (as could be passed to the JS "
                            "RegExp constructor) to test against URLs in the reftest manifest; "
                            "only test items that have a matching test URL will be run.")
     defaults["filter"] = None
 
+    self.add_option("--shuffle",
+                    action = "store_true", dest = "shuffle",
+                    help = "run reftests in random order")
+    defaults["shuffle"] = False
+
     self.add_option("--focus-filter-mode",
                     action = "store", type = "string", dest = "focusFilterMode",
                     help = "filters tests to run by whether they require focus. "
                            "Valid values are `all', `needs-focus', or `non-needs-focus'. "
                            "Defaults to `all'.")
     defaults["focusFilterMode"] = "all"
 
     self.add_option("--e10s",