Bug 992911 - (run-by-dir) add the ability to run mochitests per directory in a loop. r=ahal
authorJoel Maher <jmaher@mozilla.com>
Tue, 03 Jun 2014 11:19:28 -0400
changeset 205583 272c37660666f63c10e2cfe662e0e3df46beab26
parent 205582 dfd6b607988d9336670af8fc686d5d70a1269e32
child 205584 7652c6ab1d362fa23ea2662e09e716c873364558
push id3741
push userasasaki@mozilla.com
push dateMon, 21 Jul 2014 20:25:18 +0000
treeherdermozilla-beta@4d6f46f5af68 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs992911
milestone32.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 992911 - (run-by-dir) add the ability to run mochitests per directory in a loop. r=ahal
browser/devtools/debugger/test/head.js
build/automation.py.in
testing/mochitest/chrome-harness.js
testing/mochitest/mach_commands.py
testing/mochitest/mochitest_options.py
testing/mochitest/runtests.py
testing/mochitest/runtestsremote.py
--- a/browser/devtools/debugger/test/head.js
+++ b/browser/devtools/debugger/test/head.js
@@ -46,17 +46,20 @@ registerCleanupFunction(function() {
 
   // Debugger tests use a lot of memory, so force a GC to help fragmentation.
   info("Forcing GC after debugger test.");
   Cu.forceGC();
 });
 
 // Import the GCLI test helper
 let testDir = gTestPath.substr(0, gTestPath.lastIndexOf("/"));
-Services.scriptloader.loadSubScript(testDir + "../../../commandline/test/helpers.js", this);
+testDir = testDir.replace(/\/\//g, '/');
+testDir = testDir.replace("chrome:/mochitest", "chrome://mochitest");
+let helpersjs = testDir + "/../../commandline/test/helpers.js";
+Services.scriptloader.loadSubScript(helpersjs, this);
 
 // Redeclare dbg_assert with a fatal behavior.
 function dbg_assert(cond, e) {
   if (!cond) {
     throw e;
   }
 }
 
--- a/build/automation.py.in
+++ b/build/automation.py.in
@@ -809,17 +809,17 @@ class Automation(object):
   def checkForCrashes(self, minidumpDir, symbolsPath):
     return mozcrash.check_for_crashes(minidumpDir, symbolsPath, test_name=self.lastTestSeen)
 
   def runApp(self, testURL, env, app, profileDir, extraArgs,
              runSSLTunnel = False, utilityPath = None,
              xrePath = None, certPath = None,
              debuggerInfo = None, symbolsPath = None,
              timeout = -1, maxTime = None, onLaunch = None,
-             webapprtChrome = False, screenshotOnFail=False):
+             webapprtChrome = False, screenshotOnFail=False, testPath=None):
     """
     Run the app, log the duration it took to execute, return the status code.
     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
     """
 
     if utilityPath == None:
       utilityPath = self.DIST_BIN
     if xrePath == None:
--- a/testing/mochitest/chrome-harness.js
+++ b/testing/mochitest/chrome-harness.js
@@ -158,21 +158,22 @@ function zList(base, zReader, baseJarNam
  *  single test: [json object, path to test]
  *  list of tests: [json object, null] <- directory [heirarchy]
  */
 function getFileListing(basePath, testPath, dir, srvScope)
 {
   var uri = getResolvedURI(basePath);
   var chromeDir = getChromeDir(uri);
   chromeDir.appendRelativePath(dir);
-  basePath += '/' + dir;
+  basePath += '/' + dir.replace(/\\/g, '/');
 
   if (testPath == "false" || testPath == false) {
     testPath = "";
   }
+  testPath = testPath.replace(/\\\\/g, '\\').replace(/\\/g, '/');
 
   var ioSvc = Components.classes["@mozilla.org/network/io-service;1"].
               getService(Components.interfaces.nsIIOService);
   var testsDirURI = ioSvc.newFileURI(chromeDir);
   var testsDir = ioSvc.newURI(testPath, null, testsDirURI)
                   .QueryInterface(Components.interfaces.nsIFileURL).file;
 
   if (testPath != undefined) {
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -185,17 +185,17 @@ class MochitestRunner(MozbuildObject):
 
     def run_desktop_test(self, context, suite=None, test_paths=None, debugger=None,
         debugger_args=None, slowscript=False, screenshot_on_fail = False, shuffle=False, keep_open=False,
         rerun_failures=False, no_autorun=False, repeat=0, run_until_failure=False,
         slow=False, chunk_by_dir=0, total_chunks=None, this_chunk=None,
         jsdebugger=False, debug_on_failure=False, start_at=None, end_at=None,
         e10s=False, dmd=False, dump_output_directory=None,
         dump_about_memory_after_test=False, dump_dmd_after_test=False,
-        install_extension=None, quiet=False, environment=[], app_override=None,
+        install_extension=None, quiet=False, environment=[], app_override=None, runByDir=False,
         useTestMediaDevices=False, **kwargs):
         """Runs a mochitest.
 
         test_paths are path to tests. They can be a relative path from the
         top source directory, an absolute filename, or a directory containing
         test files.
 
         suite is the type of mochitest to run. It can be one of ('plain',
@@ -311,16 +311,17 @@ class MochitestRunner(MozbuildObject):
         options.startAt = start_at
         options.endAt = end_at
         options.e10s = e10s
         options.dumpAboutMemoryAfterTest = dump_about_memory_after_test
         options.dumpDMDAfterTest = dump_dmd_after_test
         options.dumpOutputDirectory = dump_output_directory
         options.quiet = quiet
         options.environment = environment
+        options.runByDir = runByDir
         options.useTestMediaDevices = useTestMediaDevices
 
         options.failureFile = failure_file_path
         if install_extension != None:
             options.extensionsToInstall = [os.path.join(self.topsrcdir,install_extension)]
 
         for k, v in kwargs.iteritems():
             setattr(options, k, v)
@@ -521,16 +522,22 @@ def MochitestCommand(func):
         help='Do not print test log lines unless a failure occurs.')
     func = quiet(func)
 
     setenv = CommandArgument('--setenv', default=[], action='append',
                              metavar='NAME=VALUE', dest='environment',
                              help="Sets the given variable in the application's environment")
     func = setenv(func)
 
+    runbydir = CommandArgument('--run-by-dir', default=False,
+                                 action='store_true',
+                                 dest='runByDir',
+        help='Run each directory in a single browser instance with a fresh profile.')
+    func = runbydir(func)
+
     test_media = CommandArgument('--use-test-media-devices', default=False,
                                  action='store_true',
                                  dest='useTestMediaDevices',
         help='Use test media device drivers for media testing.')
     func = test_media(func)
 
     app_override = CommandArgument('--app-override', default=None, action='store',
         help="Override the default binary used to run tests with the path you provide, e.g. " \
--- a/testing/mochitest/mochitest_options.py
+++ b/testing/mochitest/mochitest_options.py
@@ -94,16 +94,22 @@ class MochitestOptions(optparse.OptionPa
           "default": None,
         }],
         [["--chunk-by-dir"],
         { "type": "int",
           "dest": "chunkByDir",
           "help": "group tests together in the same chunk that are in the same top chunkByDir directories",
           "default": 0,
         }],
+        [["--run-by-dir"],
+        { "action": "store_true",
+          "dest": "runByDir",
+          "help": "Run each directory in a single browser instance with a fresh profile",
+          "default": False,
+        }],
         [["--shuffle"],
         { "dest": "shuffle",
           "action": "store_true",
           "help": "randomize test order",
           "default": False,
         }],
         [["--console-level"],
         { "action": "store",
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -412,17 +412,17 @@ class MochitestUtilsMixin(object):
 
     pathPieces = filename.split("/")
 
     return (testPattern.match(pathPieces[-1]) and
             not re.search(r'\^headers\^$', filename))
 
   def getTestPath(self, options):
     if options.ipcplugins:
-      return "dom/plugins/test"
+      return "dom/plugins/test/mochitest"
     else:
       return options.testPath
 
   def getTestRoot(self, options):
     if options.browserChrome:
       if options.immersiveMode:
         return 'metro'
       return 'browser'
@@ -446,64 +446,48 @@ class MochitestUtilsMixin(object):
       testURL = "about:blank"
     return testURL
 
   def buildTestPath(self, options):
     """ Build the url path to the specific test harness and test file or directory
         Build a manifest of tests to run and write out a json file for the harness to read
     """
     manifest = None
-
-    testRoot = self.getTestRoot(options)
-    # testdir refers to 'mochitest' here.
-    testdir = SCRIPT_DIR.split(os.getcwd())[-1]
-    testdir = testdir.strip(os.sep)
-    testRootAbs = os.path.abspath(os.path.join(testdir, testRoot))
-    if isinstance(options.manifestFile, TestManifest):
-        manifest = options.manifestFile
-    elif options.manifestFile and os.path.isfile(options.manifestFile):
-      manifestFileAbs = os.path.abspath(options.manifestFile)
-      assert manifestFileAbs.startswith(testRootAbs)
-      manifest = TestManifest([options.manifestFile], strict=False)
-    else:
-      masterName = self.getTestFlavor(options) + '.ini'
-      masterPath = os.path.join(testdir, testRoot, masterName)
-
-      if os.path.exists(masterPath):
-        manifest = TestManifest([masterPath], strict=False)
+    manifest = self.getTestManifest(options)
 
     if manifest:
       # Python 2.6 doesn't allow unicode keys to be used for keyword
       # arguments. This gross hack works around the problem until we
       # rid ourselves of 2.6.
       info = {}
       for k, v in mozinfo.info.items():
         if isinstance(k, unicode):
           k = k.encode('ascii')
         info[k] = v
 
       # Bug 883858 - return all tests including disabled tests
       testPath = self.getTestPath(options)
+      testPath = testPath.replace('\\', '/')
       if testPath.endswith('.html') or \
          testPath.endswith('.xhtml') or \
          testPath.endswith('.xul') or \
          testPath.endswith('.js'):
           # In the case where we have a single file, we don't want to filter based on options such as subsuite.
           tests = manifest.active_tests(disabled=True, options=None, **info)
           for test in tests:
               if 'disabled' in test:
                   del test['disabled']
       else:
           tests = manifest.active_tests(disabled=True, options=options, **info)
       paths = []
 
       for test in tests:
         pathAbs = os.path.abspath(test['path'])
-        assert pathAbs.startswith(testRootAbs)
-        tp = pathAbs[len(testRootAbs):].replace('\\', '/').strip('/')
+        assert pathAbs.startswith(self.testRootAbs)
+        tp = pathAbs[len(self.testRootAbs):].replace('\\', '/').strip('/')
 
         # Filter out tests if we are using --test-path
         if testPath and not tp.startswith(testPath):
           continue
 
         if not self.isTest(options, tp):
           print 'Warning: %s from manifest %s is not a valid test' % (test['name'], test['manifest'])
           continue
@@ -517,17 +501,17 @@ class MochitestUtilsMixin(object):
       def path_sort(ob1, ob2):
         path1 = ob1['path'].split('/')
         path2 = ob2['path'].split('/')
         return cmp(path1, path2)
 
       paths.sort(path_sort)
 
       # Bug 883865 - add this functionality into manifestDestiny
-      with open(os.path.join(testdir, 'tests.json'), 'w') as manifestFile:
+      with open(os.path.join(SCRIPT_DIR, 'tests.json'), 'w') as manifestFile:
         manifestFile.write(json.dumps({'tests': paths}))
       options.manifestFile = 'tests.json'
 
     return self.buildTestURL(options)
 
   def startWebSocketServer(self, options, debuggerInfo):
     """ Launch the websocket server """
     self.wsserver = WebSocketServer(options, SCRIPT_DIR, debuggerInfo)
@@ -1060,19 +1044,20 @@ class Mochitest(MochitestUtilsMixin):
       browserEnv["NSPR_LOG_FILE"] = "%s/nspr.log" % tempfile.gettempdir()
       browserEnv["GECKO_SEPARATE_NSPR_LOGS"] = "1"
 
     if debugger and not options.slowscript:
       browserEnv["JS_DISABLE_SLOW_SCRIPT_SIGNALS"] = "1"
 
     return browserEnv
 
-  def cleanup(self, manifest, options):
+  def cleanup(self, options):
     """ remove temporary files and profile """
-    os.remove(manifest)
+    if self.manifest is not None:
+      os.remove(self.manifest)
     del self.profile
     if options.pidFile != "":
       try:
         os.remove(options.pidFile)
         if os.path.exists(options.pidFile + ".xpcshell.pid"):
           os.remove(options.pidFile + ".xpcshell.pid")
       except:
         log.warn("cleaning up pidfile '%s' was unsuccessful from the test harness", options.pidFile)
@@ -1181,17 +1166,18 @@ class Mochitest(MochitestUtilsMixin):
              profile,
              extraArgs,
              utilityPath,
              debuggerInfo=None,
              symbolsPath=None,
              timeout=-1,
              onLaunch=None,
              webapprtChrome=False,
-             screenshotOnFail=False):
+             screenshotOnFail=False,
+             testPath=None):
     """
     Run the app, log the duration it took to execute, return the status code.
     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
     """
 
     # debugger information
     interactive = False
     debug_args = None
@@ -1245,17 +1231,17 @@ class Mochitest(MochitestUtilsMixin):
                                          symbolsPath=symbolsPath,
                                          dump_screen_on_timeout=not debuggerInfo,
                                          dump_screen_on_fail=screenshotOnFail,
                                          shutdownLeaks=shutdownLeaks,
         )
 
       def timeoutHandler():
         browserProcessId = outputHandler.browserProcessId
-        self.handleTimeout(timeout, proc, utilityPath, debuggerInfo, browserProcessId)
+        self.handleTimeout(timeout, proc, utilityPath, debuggerInfo, browserProcessId, testPath)
       kp_kwargs = {'kill_on_timeout': False,
                    'cwd': SCRIPT_DIR,
                    'onTimeout': [timeoutHandler]}
       kp_kwargs['processOutputLine'] = [outputHandler]
 
       # create mozrunner instance and start the system under test process
       self.lastTestSeen = self.test_name
       startTime = datetime.now()
@@ -1334,16 +1320,68 @@ class Mochitest(MochitestUtilsMixin):
       if os.path.exists(processLog):
         os.remove(processLog)
 
     return status
 
   def runTests(self, options, onLaunch=None):
     """ Prepare, configure, run tests and cleanup """
 
+    # Create variables to count the number of passes, fails, todos.
+    self.countpass = 0
+    self.countfail = 0
+    self.counttodo = 0
+
+    self.testRoot = self.getTestRoot(options)
+    self.testRootAbs = os.path.join(SCRIPT_DIR, self.testRoot)
+
+    if not options.runByDir:
+      return self.doTests(options, onLaunch)
+
+    dirs = self.getDirectories(options)
+    
+    if options.totalChunks > 1:
+      chunkSize = int(len(dirs) / options.totalChunks) + 1
+      start = chunkSize * (options.thisChunk-1)
+      end = chunkSize * (options.thisChunk)
+      dirs = dirs[start:end]
+
+    options.totalChunks = None
+    options.thisChunk = None
+    options.chunkByDir = 0
+    inputTestPath = self.getTestPath(options)
+    for dir in dirs:
+      options.manifestFile = None
+
+      if inputTestPath and not inputTestPath.startswith(dir):
+        continue
+
+      options.testPath = dir
+      print "testpath: %s" % options.testPath
+
+      options.profilePath = tempfile.mkdtemp()
+      self.urlOpts = []
+      self.doTests(options, onLaunch)
+
+    # printing total number of tests
+    if options.browserChrome:
+      print "TEST-INFO | checking window state"
+      print "Browser Chrome Test Summary"
+      print "\tPassed: %s" % self.countpass
+      print "\tFailed: %s" % self.countfail
+      print "\tTodo: %s" % self.counttodo
+      print "*** End BrowserChrome Test Results ***"
+    else:
+      print "0 INFO TEST-START | Shutdown"
+      print "1 INFO Passed:  %s" % self.countpass
+      print "2 INFO Failed:  %s" % self.countfail
+      print "3 INFO Todo:    %s" % self.counttodo
+      print "4 INFO SimpleTest FINISHED"
+
+  def doTests(self, options, onLaunch=None):
     # get debugger info, a dict of:
     # {'path': path to the debugger (string),
     #  'interactive': whether the debugger is interactive or not (bool)
     #  'args': arguments to the debugger (list)
     # TODO: use mozrunner.local.debugger_arguments:
     # https://github.com/mozilla/mozbase/blob/master/mozrunner/mozrunner/local.py#L42
     debuggerInfo = getDebuggerInfo(self.oldcwd,
                                    options.debugger,
@@ -1354,32 +1392,43 @@ class Mochitest(MochitestUtilsMixin):
       devices = findTestMediaDevices()
       if not devices:
         log.error("Could not find test media devices to use")
         return 1
       self.mediaDevices = devices
 
     self.leak_report_file = os.path.join(options.profilePath, "runtests_leaks.log")
 
-    browserEnv = self.buildBrowserEnv(options, debuggerInfo is not None)
-    if browserEnv is None:
+    self.browserEnv = self.buildBrowserEnv(options, debuggerInfo is not None)
+    if self.browserEnv is None:
       return 1
 
     # buildProfile sets self.profile .
     # This relies on sideeffects and isn't very stateful:
     # https://bugzilla.mozilla.org/show_bug.cgi?id=919300
-    manifest = self.buildProfile(options)
-    if manifest is None:
+    self.manifest = self.buildProfile(options)
+    if self.manifest is None:
       return 1
 
     try:
       self.startServers(options, debuggerInfo)
 
       testURL = self.buildTestPath(options)
-      self.buildURLOptions(options, browserEnv)
+
+      # read the number of tests here, if we are not going to run any, terminate early
+      if os.path.exists(os.path.join(SCRIPT_DIR, 'tests.json')):
+        with open(os.path.join(SCRIPT_DIR, 'tests.json')) as fHandle:
+          tests = json.load(fHandle)
+        count = 0
+        for test in tests['tests']:
+          count += 1
+        if count == 0:
+          return 1
+
+      self.buildURLOptions(options, self.browserEnv)
       if self.urlOpts:
         testURL += "?" + "&".join(self.urlOpts)
 
       if options.webapprtContent:
         options.browserArgs.extend(('-test-mode', testURL))
         testURL = None
 
       if options.immersiveMode:
@@ -1403,27 +1452,28 @@ class Mochitest(MochitestUtilsMixin):
         timeout = 330.0 # default JS harness timeout is 300 seconds
 
       if options.vmwareRecording:
         self.startVMwareRecording(options);
 
       log.info("runtests.py | Running tests: start.\n")
       try:
         status = self.runApp(testURL,
-                             browserEnv,
+                             self.browserEnv,
                              options.app,
                              profile=self.profile,
                              extraArgs=options.browserArgs,
                              utilityPath=options.utilityPath,
                              debuggerInfo=debuggerInfo,
                              symbolsPath=options.symbolsPath,
                              timeout=timeout,
                              onLaunch=onLaunch,
                              webapprtChrome=options.webapprtChrome,
-                             screenshotOnFail=options.screenshotOnFail
+                             screenshotOnFail=options.screenshotOnFail,
+                             testPath=options.testPath
         )
       except KeyboardInterrupt:
         log.info("runtests.py | Received keyboard interrupt.\n");
         status = -1
       except:
         traceback.print_exc()
         log.error("Automation Error: Received unexpected exception while running application\n")
         status = 1
@@ -1438,25 +1488,28 @@ class Mochitest(MochitestUtilsMixin):
     if self.nsprLogs:
       with zipfile.ZipFile("%s/nsprlog.zip" % browserEnv["MOZ_UPLOAD_DIR"], "w", zipfile.ZIP_DEFLATED) as logzip:
         for logfile in glob.glob("%s/nspr*.log*" % tempfile.gettempdir()):
           logzip.write(logfile)
           os.remove(logfile)
 
     log.info("runtests.py | Running tests: end.")
 
-    if manifest is not None:
-      self.cleanup(manifest, options)
+    if self.manifest is not None:
+      self.cleanup(options)
 
     return status
 
-  def handleTimeout(self, timeout, proc, utilityPath, debuggerInfo, browserProcessId):
+  def handleTimeout(self, timeout, proc, utilityPath, debuggerInfo, browserProcessId, testPath=None):
     """handle process output timeout"""
     # TODO: bug 913975 : _processOutput should call self.processOutputLine one more time one timeout (I think)
-    log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output", self.lastTestSeen, int(timeout))
+    if testPath:
+      log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output on %s", self.lastTestSeen, int(timeout), testPath)
+    else:
+      log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output", self.lastTestSeen, int(timeout))
     browserProcessId = browserProcessId or proc.pid
     self.killAndGetStack(browserProcessId, utilityPath, debuggerInfo, dump_screen=not debuggerInfo)
 
   ### output processing
 
   class OutputHandler(object):
     """line output handler for mozrunner"""
     def __init__(self, harness, utilityPath, symbolsPath=None, dump_screen_on_timeout=True, dump_screen_on_fail=False, shutdownLeaks=None):
@@ -1493,16 +1546,17 @@ class Mochitest(MochitestUtilsMixin):
       return [self.fix_stack,
               self.format,
               self.record_last_test,
               self.dumpScreenOnTimeout,
               self.dumpScreenOnFail,
               self.metro_subprocess_id,
               self.trackShutdownLeaks,
               self.log,
+              self.countline,
               ]
 
     def stackFixer(self):
       """
       return 2-tuple, (stackFixerFunction, StackFixerProcess),
       if any, to use on the output lines
       """
 
@@ -1545,16 +1599,30 @@ class Mochitest(MochitestUtilsMixin):
           log.info("TEST-UNEXPECTED-FAIL | runtests.py | Stack fixer process exited with code %d during test run", status)
 
       if self.shutdownLeaks:
         self.shutdownLeaks.process()
 
 
     # output line handlers:
     # these take a line and return a line
+    def countline(self, line):
+      val = 0
+      try:
+        val = int(line.split(':')[-1].strip())
+      except ValueError, e:
+        return line
+
+      if "Passed:" in line:
+        self.harness.countpass += val
+      elif "Failed:" in line:
+        self.harness.countfail += val
+      elif "Todo:" in line:
+        self.harness.counttodo += val
+      return line
 
     def fix_stack(self, line):
       if self.stackFixerFunction:
         return self.stackFixerFunction(line)
       return line
 
     def format(self, line):
       """format the line"""
@@ -1594,23 +1662,22 @@ class Mochitest(MochitestUtilsMixin):
       log.info(line)
       return line
 
 
   def makeTestConfig(self, options):
     "Creates a test configuration file for customizing test execution."
     options.logFile = options.logFile.replace("\\", "\\\\")
     options.testPath = options.testPath.replace("\\", "\\\\")
-    testRoot = self.getTestRoot(options)
 
     if "MOZ_HIDE_RESULTS_TABLE" in os.environ and os.environ["MOZ_HIDE_RESULTS_TABLE"] == "1":
       options.hideResultsTable = True
 
     d = dict(options.__dict__)
-    d['testRoot'] = testRoot
+    d['testRoot'] = self.testRoot
     content = json.dumps(d)
 
     with open(os.path.join(options.profilePath, "testConfig.js"), "w") as config:
       config.write(content)
 
   def installExtensionFromPath(self, options, path, extensionID = None):
     """install an extension to options.profilePath"""
 
@@ -1636,16 +1703,61 @@ class Mochitest(MochitestUtilsMixin):
 
     addons.install_from_path(path)
 
   def installExtensionsToProfile(self, options):
     "Install special testing extensions, application distributed extensions, and specified on the command line ones to testing profile."
     for path in self.getExtensionsToInstall(options):
       self.installExtensionFromPath(options, path)
 
+  def getTestManifest(self, options):
+    if isinstance(options.manifestFile, TestManifest):
+        manifest = options.manifestFile
+    elif options.manifestFile and os.path.isfile(options.manifestFile):
+      manifestFileAbs = os.path.abspath(options.manifestFile)
+      assert manifestFileAbs.startswith(SCRIPT_DIR)
+      manifest = TestManifest([options.manifestFile], strict=False)
+    elif options.manifestFile and os.path.isfile(os.path.join(SCRIPT_DIR, options.manifestFile)):
+      manifestFileAbs = os.path.abspath(os.path.join(SCRIPT_DIR, options.manifestFile))
+      assert manifestFileAbs.startswith(SCRIPT_DIR)
+      manifest = TestManifest([manifestFileAbs], strict=False)
+    else:
+      masterName = self.getTestFlavor(options) + '.ini'
+      masterPath = os.path.join(SCRIPT_DIR, self.testRoot, masterName)
+
+      if os.path.exists(masterPath):
+        manifest = TestManifest([masterPath], strict=False)
+
+    return manifest
+
+  def getDirectories(self, options):
+    """
+        Make the list of directories by parsing manifests
+    """
+    info = {}
+    for k, v in mozinfo.info.items():
+      if isinstance(k, unicode):
+        k = k.encode('ascii')
+      info[k] = v
+
+    dirlist = []
+
+    manifest = self.getTestManifest(options)
+    tests = manifest.active_tests(disabled=False, options=options, **info)
+    for test in tests:
+      pathAbs = os.path.abspath(test['path'])
+      assert pathAbs.startswith(self.testRootAbs)
+      tp = pathAbs[len(self.testRootAbs):].replace('\\', '/').strip('/')
+
+      rootdir = '/'.join(tp.split('/')[:-1])
+      if rootdir not in dirlist:
+        dirlist.append(rootdir)
+
+    dirlist.sort()
+    return dirlist
 
 def main():
 
   # parse command line options
   mochitest = Mochitest()
   parser = MochitestOptions()
   options, args = parser.parse_args()
   options = parser.verifyOptions(options, mochitest)
--- a/testing/mochitest/runtestsremote.py
+++ b/testing/mochitest/runtestsremote.py
@@ -234,26 +234,25 @@ class MochiRemote(Mochitest):
         self.environment = self._automation.environment
         self.remoteProfile = options.remoteTestRoot + "/profile"
         self._automation.setRemoteProfile(self.remoteProfile)
         self.remoteLog = options.remoteLogFile
         self.localLog = options.logFile
         self._automation.deleteANRs()
         self.certdbNew = True
 
-    def cleanup(self, manifest, options):
+    def cleanup(self, options):
         if self._dm.fileExists(self.remoteLog):
             self._dm.getFile(self.remoteLog, self.localLog)
             self._dm.removeFile(self.remoteLog)
         else:
             log.warn("Unable to retrieve log file (%s) from remote device",
                 self.remoteLog)
         self._dm.removeDir(self.remoteProfile)
-        if manifest is not None:
-            Mochitest.cleanup(self, manifest, options)
+        Mochitest.cleanup(self, options)
 
     def findPath(self, paths, filename = None):
         for path in paths:
             p = path
             if filename:
                 p = os.path.join(p, filename)
             if os.path.exists(self.getFullPath(p)):
                 return path
@@ -707,17 +706,17 @@ def main():
                 # Ensure earlier failures aren't overwritten by success on this run
                 if retVal is None or retVal == 0:
                     retVal = result
             except:
                 log.error("Automation Error: Exception caught while running tests")
                 traceback.print_exc()
                 mochitest.stopServers()
                 try:
-                    mochitest.cleanup(None, options)
+                    mochitest.cleanup(options)
                 except devicemanager.DMError:
                     # device error cleaning up... oh well!
                     pass
                 retVal = 1
                 break
             finally:
                 # Clean-up added bookmarks
                 if test['name'] == "testImportFromAndroid":
@@ -742,17 +741,17 @@ def main():
         try:
             dm.recordLogcat()
             retVal = mochitest.runTests(options)
         except:
             log.error("Automation Error: Exception caught while running tests")
             traceback.print_exc()
             mochitest.stopServers()
             try:
-                mochitest.cleanup(None, options)
+                mochitest.cleanup(options)
             except devicemanager.DMError:
                 # device error cleaning up... oh well!
                 pass
             retVal = 1
 
     mochitest.printDeviceInfo(printLogcat=True)
 
     sys.exit(retVal)