Bug 687264 - add mochitest support for filtering tests. r=ctalbert
authorJoel Maher <jmaher@mozilla.com>
Mon, 26 Sep 2011 07:41:17 -0400
changeset 77573 0c880a09186d4afeddeda29bc191a1691e86804a
parent 77568 7d983fc8a13c41fef193013d146474284bc4e877
child 77574 3995f7c64af8de1a7259c3a89b12d94f825caa35
push id21212
push userb56girard@gmail.com
push dateMon, 26 Sep 2011 19:53:41 +0000
treeherdermozilla-central@24bc89c8bcbe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersctalbert
bugs687264
milestone9.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 687264 - add mochitest support for filtering tests. r=ctalbert
testing/mochitest/runtests.py
testing/mochitest/tests/SimpleTest/setup.js
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -230,20 +230,29 @@ class MochitestOptions(optparse.OptionPa
                     action = "store_true", dest = "vmwareRecording",
                     help = "enables recording while the application is running "
                            "inside a VMware Workstation 7.0 or later VM")
     defaults["vmwareRecording"] = False
 
     self.add_option("--repeat",
                     action = "store", type = "int",
                     dest = "repeat", metavar = "REPEAT",
-                    help = "repeats the test or set of tests the given number of times, ie: repeat=1 will run the test twice.")
-                   
+                    help = "repeats the test or set of tests the given number of times, ie: repeat=1 will run the test twice.")                   
     defaults["repeat"] = 0
 
+    self.add_option("--run-only-tests",
+                    action = "store", type="string", dest = "runOnlyTests",
+                    help = "JSON list of tests that we only want to run, cannot be specified with --exclude-tests.")
+    defaults["runOnlyTests"] = None
+
+    self.add_option("--exclude-tests",
+                    action = "store", type="string", dest = "excludeTests",
+                    help = "JSON list of tests that we want to not run, cannot be specified with --run-only-tests.")
+    defaults["excludeTests"] = None
+
     # -h, --help are automatically handled by OptionParser
 
     self.set_defaults(**defaults)
 
     usage = """\
 Usage instructions for runtests.py.
 All arguments are optional.
 If --chrome is specified, chrome tests will be run instead of web content tests.
@@ -297,16 +306,27 @@ See <http://mochikit.com/doc/html/MochiK
       if not self._automation.IS_WIN32:
         self.error("use-vmware-recording is only supported on Windows.")
       mochitest.vmwareHelperPath = os.path.join(
         options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll")
       if not os.path.exists(mochitest.vmwareHelperPath):
         self.error("%s not found, cannot automate VMware recording." %
                    mochitest.vmwareHelperPath)
 
+    if options.runOnlyTests != None and options.excludeTests != None:
+      self.error("We can only support --run-only-tests OR --exclude-tests, not both.")
+      
+    if (options.runOnlyTests):
+      if (not os.path.exists(os.path.join(os.path.dirname(__file__), options.runOnlyTests))):
+        self.error("unable to find --run-only-tests file '%s'" % (options.runOnlyTests));
+        
+    if (options.excludeTests):
+      if (not os.path.exists(os.path.join(os.path.dirname(__file__), options.excludeTests))):
+        self.error("unable to find --exclude-tests file '%s'" % (options.excludeTests));
+
     return options
 
 
 #######################
 # HTTP SERVER SUPPORT #
 #######################
 
 class MochitestServer:
@@ -576,16 +596,20 @@ class Mochitest(object):
       if options.shuffle:
         self.urlOpts.append("shuffle=1")
       if "MOZ_HIDE_RESULTS_TABLE" in env and env["MOZ_HIDE_RESULTS_TABLE"] == "1":
         self.urlOpts.append("hideResultsTable=1")
       if options.repeat:
         self.urlOpts.append("repeat=%d" % options.repeat)
       if os.path.isfile(os.path.join(self.oldcwd, os.path.dirname(__file__), self.TEST_PATH, options.testPath)) and options.repeat > 0:
         self.urlOpts.append("testname=%s" % ("/").join([self.TEST_PATH, options.testPath]))
+      if options.runOnlyTests:
+        self.urlOpts.append("runOnlyTests=%s" % options.runOnlyTests)
+      elif options.excludeTests:
+        self.urlOpts.append("excludeTests=%s" % options.excludeTests)
 
   def cleanup(self, manifest, options):
     """ remove temporary files and profile """
     os.remove(manifest)
     shutil.rmtree(options.profilePath)
 
   def startVMwareRecording(self, options):
     """ starts recording inside VMware VM using the recording helper dll """
--- a/testing/mochitest/tests/SimpleTest/setup.js
+++ b/testing/mochitest/tests/SimpleTest/setup.js
@@ -130,16 +130,20 @@ if (!params.quiet) {
     dump(msg.num + " " + msg.level + " " + msg.info.join(' ') + "\n");
   }
   TestRunner.logger.addListener("dumpListener", consoleLevel + "", dumpListener);
 }
 
 var gTestList = [];
 var RunSet = {}
 RunSet.runall = function(e) {
+  // Filter tests to include|exclude tests based on data in params.filter.
+  // This allows for including or excluding tests from the gTestList
+  gTestList = filterTests(params.runOnlyTests, params.excludeTests);
+
   // Which tests we're going to run
   var my_tests = gTestList;
 
   if (params.totalChunks && params.thisChunk) {
     var total_chunks = parseInt(params.totalChunks);
     // this_chunk is in the range [1,total_chunks]
     var this_chunk = parseInt(params.thisChunk);
 
@@ -208,16 +212,73 @@ RunSet.reloadAndRunAll = function(e) {
     window.location.href = window.location.href;
   } else if (window.location.search) {
     window.location.href += "&autorun=1";
   } else {
     window.location.href += "?autorun=1";
   }  
 };
 
+// Test Filtering Code
+
+// Open the file referenced by runOnly|exclude and use that to compare against
+// gTestList.  Return a modified version of gTestList
+function filterTests(runOnly, exclude) {
+  var filteredTests = [];
+  var filterFile = null;
+
+  if (runOnly) {
+    filterFile = runOnly;
+  } else if (exclude) {
+    filterFile = exclude;
+  }
+
+  if (filterFile == null)
+    return gTestList;
+
+  var datafile = "http://mochi.test:8888/" + filterFile;
+  var objXml = new XMLHttpRequest();
+  objXml.open("GET",datafile,false);
+  objXml.send(null);
+  try {
+    var filter = JSON.parse(objXml.responseText);
+  } catch (ex) {
+    dump("INFO | setup.js | error loading or parsing '" + datafile + "'\n");
+    return gTestList;
+  }
+  
+  for (var i = 0; i < gTestList.length; ++i) {
+    var test_path = gTestList[i];
+    
+    //We use tmp_path to remove leading '/'
+    var tmp_path = test_path.replace(/^\//, '');
+
+    var found = false;
+
+    for (var f in filter) {
+      // Remove leading /tests/ if exists
+      file = f.replace(/^\//, '')
+      file = file.replace(/^tests\//, '')
+      
+      // Match directory or filename, gTestList has tests/<path>
+      if (tmp_path.match("^tests/" + file) != null) {
+        if (runOnly)
+          filteredTests.push(test_path);
+        found = true;
+        break;
+      }
+    }
+
+    if (exclude && !found)
+      filteredTests.push(test_path);
+  }
+
+  return filteredTests;
+}
+
 // UI Stuff
 function toggleVisible(elem) {
     toggleElementClass("invisible", elem);
 }
 
 function makeVisible(elem) {
     removeElementClass(elem, "invisible");
 }