Bug 1131098 - Make mochitest use manifestparser's chunking algorithms and remove JS based ones, r=jmaher
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 10 Mar 2015 09:55:30 -0400
changeset 263462 187d5f6e0b0328da86ea9df7cb399dd71f519c7e
parent 263461 abd759f41a6d479e0591a55b769ad9c7a6495863
child 263463 851347e26940f7449afc1d740bf1e3c1d1b62c2a
push id830
push userraliiev@mozilla.com
push dateFri, 19 Jun 2015 19:24:37 +0000
treeherdermozilla-release@932614382a68 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1131098
milestone39.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1131098 - Make mochitest use manifestparser's chunking algorithms and remove JS based ones, r=jmaher
addon-sdk/source/test/addons/jetpack-addon.ini
testing/mochitest/bisection.py
testing/mochitest/browser-harness.xul
testing/mochitest/chunkifyTests.js
testing/mochitest/jetpack-addon-harness.js
testing/mochitest/jetpack-package-harness.js
testing/mochitest/manifestLibrary.js
testing/mochitest/runtests.py
testing/mochitest/runtestsremote.py
testing/mochitest/tests/SimpleTest/setup.js
testing/mozbase/manifestparser/manifestparser/filters.py
--- a/addon-sdk/source/test/addons/jetpack-addon.ini
+++ b/addon-sdk/source/test/addons/jetpack-addon.ini
@@ -8,17 +8,16 @@
 [developers.xpi]
 [e10s.xpi]
 skip-if = true
 [e10s-content.xpi]
 skip-if = true
 [e10s-tabs.xpi]
 skip-if = true
 [l10n.xpi]
-[l10n-manifest.xpi]
 [l10n-properties.xpi]
 [layout-change.xpi]
 [main.xpi]
 [name-in-numbers.xpi]
 [name-in-numbers-plus.xpi]
 [packaging.xpi]
 [packed.xpi]
 [page-mod-debugger-post.xpi]
--- a/testing/mochitest/bisection.py
+++ b/testing/mochitest/bisection.py
@@ -26,63 +26,18 @@ class Bisect(object):
         self.contents['loop'] = 0
         return status
 
     def reset(self, expectedError, result):
         "This method is used to initialize self.expectedError and self.result for each loop in runtests."
         self.expectedError = expectedError
         self.result = result
 
-    def get_test_chunk(self, options, tests):
-        "This method is used to return the chunk of test that is to be run"
-        if not options.totalChunks or not options.thisChunk:
-            return tests
-
-        # The logic here is same as chunkifyTests.js, we need this for
-        # bisecting tests.
-        if options.chunkByDir:
-            tests_by_dir = {}
-            test_dirs = []
-            for test in tests:
-                directory = test.split("/")
-                directory = directory[
-                    0:min(
-                        options.chunkByDir,
-                        len(directory) -
-                        1)]
-                directory = "/".join(directory)
-
-                if directory not in tests_by_dir:
-                    tests_by_dir[directory] = [test]
-                    test_dirs.append(directory)
-                else:
-                    tests_by_dir[directory].append(test)
-
-            tests_per_chunk = float(len(test_dirs)) / options.totalChunks
-            start = int(round((options.thisChunk - 1) * tests_per_chunk))
-            end = int(round((options.thisChunk) * tests_per_chunk))
-            test_dirs = test_dirs[start:end]
-            return_tests = []
-            for directory in test_dirs:
-                return_tests += tests_by_dir[directory]
-
-        else:
-            tests_per_chunk = float(len(tests)) / options.totalChunks
-            start = int(round((options.thisChunk - 1) * tests_per_chunk))
-            end = int(round(options.thisChunk * tests_per_chunk))
-            return_tests = tests[start:end]
-
-        options.totalChunks = None
-        options.thisChunk = None
-        options.chunkByDir = None
-        return return_tests
-
     def get_tests_for_bisection(self, options, tests):
         "Make a list of tests for bisection from a given list of tests"
-        tests = self.get_test_chunk(options, tests)
         bisectlist = []
         for test in tests:
             bisectlist.append(test)
             if test.endswith(options.bisectChunk):
                 break
 
         return bisectlist
 
--- a/testing/mochitest/browser-harness.xul
+++ b/testing/mochitest/browser-harness.xul
@@ -224,21 +224,16 @@
       var fileNames = [];
       var fileNameRegexp = /browser_.+\.js$/;
       srvScope.arrayOfTestFiles(links, fileNames, fileNameRegexp);
 
       if (gConfig.startAt || gConfig.endAt) {
         fileNames = skipTests(fileNames, gConfig.startAt, gConfig.endAt);
       }
 
-      if (gConfig.totalChunks && gConfig.thisChunk) {
-        fileNames = chunkifyTests(fileNames, gConfig.totalChunks,
-                                  gConfig.thisChunk, gConfig.chunkByDir);
-      }
-
       createTester(fileNames.map(function (f) { return new browserTest(f); }));
     }
 
     function setStatus(aStatusString) {
       document.getElementById("status").value = aStatusString;
     }
 
     function createTester(links) {
--- a/testing/mochitest/chunkifyTests.js
+++ b/testing/mochitest/chunkifyTests.js
@@ -1,84 +1,12 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-function chunkifyTests(tests, totalChunks, thisChunk, chunkByDir, logger) {
-  var total_chunks = parseInt(totalChunks);
-  // this_chunk is in the range [1,total_chunks]
-  var this_chunk = parseInt(thisChunk);
-  var returnTests;
-
-  // We want to split the tests up into chunks according to which directory
-  // they're in
-  if (chunkByDir) {
-    chunkByDir = parseInt(chunkByDir);
-    var tests_by_dir = {};
-    var test_dirs = []
-    for (var i = 0; i < tests.length; ++i) {
-      if ((tests[i] instanceof Object) && ('test' in tests[i])) {
-        var test_path = tests[i]['test']['url'];
-      }
-      else if ((tests[i] instanceof Object) && ('url' in tests[i])) {
-        // This condition is needed to run --chunk-by-dir on mochitest bc and dt.
-        var test_path = tests[i]['url'];
-      }
-      else {
-        // This condition is needed to run --chunk-by-dir on android chunks.
-        var test_path = tests[i];
-      }
-      if (test_path[0] == '/') {
-        test_path = test_path.substr(1);
-      }
-      // mochitest-chrome and mochitest-browser-chrome pass an array of chrome://
-      // URIs
-      var protocolRegexp = /^[a-zA-Z]+:\/\//;
-      if (protocolRegexp.test(test_path)) {
-        test_path = test_path.replace(protocolRegexp, "");
-      }
-      var dir = test_path.split("/");
-      // We want the first chunkByDir+1 components, or everything but the
-      // last component, whichever is less.
-      // we add 1 to chunkByDir since 'tests' is always part of the path, and
-      // want to ignore the last component since it's the test filename.
-      dir = dir.slice(0, Math.min(chunkByDir+1, dir.length-1));
-      // reconstruct a directory name
-      dir = dir.join("/");
-      if (!(dir in tests_by_dir)) {
-        tests_by_dir[dir] = [tests[i]];
-        test_dirs.push(dir);
-      } else {
-        tests_by_dir[dir].push(tests[i]);
-      }
-    }
-    var tests_per_chunk = test_dirs.length / total_chunks;
-    var start = Math.round((this_chunk-1) * tests_per_chunk);
-    var end = Math.round(this_chunk * tests_per_chunk);
-    returnTests = [];
-    var dirs = []
-    for (var i = start; i < end; ++i) {
-      var dir = test_dirs[i];
-      dirs.push(dir);
-      returnTests = returnTests.concat(tests_by_dir[dir]);
-    }
-    if (logger)
-      logger.log("Running tests in " + dirs.join(", "));
-  } else {
-    var tests_per_chunk = tests.length / total_chunks;
-    var start = Math.round((this_chunk-1) * tests_per_chunk);
-    var end = Math.round(this_chunk * tests_per_chunk);
-    returnTests = tests.slice(start, end);
-    if (logger)
-      logger.log("Running tests " + (start+1) + "-" + end + "/" + tests.length);
-  }
-
-  return returnTests;
-}
-
 function skipTests(tests, startTestPattern, endTestPattern) {
   var startIndex = 0, endIndex = tests.length - 1;
   for (var i = 0; i < tests.length; ++i) {
     var test_path;
     if ((tests[i] instanceof Object) && ('test' in tests[i])) {
       test_path = tests[i]['test']['url'];
     } else if ((tests[i] instanceof Object) && ('url' in tests[i])) {
       test_path = tests[i]['url'];
--- a/testing/mochitest/jetpack-addon-harness.js
+++ b/testing/mochitest/jetpack-addon-harness.js
@@ -148,21 +148,16 @@ function testInit() {
       let fileNames = [];
       let fileNameRegexp = /.+\.xpi$/;
       arrayOfTestFiles(links, fileNames, fileNameRegexp);
 
       if (config.startAt || config.endAt) {
         fileNames = skipTests(fileNames, config.startAt, config.endAt);
       }
 
-      if (config.totalChunks && config.thisChunk) {
-        fileNames = chunkifyTests(fileNames, config.totalChunks,
-                                  config.thisChunk, config.chunkByDir);
-      }
-
       // Override the SDK modules if necessary
       try {
         let sdklibs = Services.prefs.getCharPref("extensions.sdk.path");
         // sdkpath is a file path, make it a URI
         let sdkfile = Cc["@mozilla.org/file/local;1"].
                       createInstance(Ci.nsIFile);
         sdkfile.initWithPath(sdklibs);
         sdkpath = Services.io.newFileURI(sdkfile).spec;
--- a/testing/mochitest/jetpack-package-harness.js
+++ b/testing/mochitest/jetpack-package-harness.js
@@ -129,21 +129,16 @@ function testInit() {
       let fileNames = [];
       let fileNameRegexp = /test-.+\.js$/;
       arrayOfTestFiles(links, fileNames, fileNameRegexp);
 
       if (config.startAt || config.endAt) {
         fileNames = skipTests(fileNames, config.startAt, config.endAt);
       }
 
-      if (config.totalChunks && config.thisChunk) {
-        fileNames = chunkifyTests(fileNames, config.totalChunks,
-                                  config.thisChunk, config.chunkByDir);
-      }
-
       // The SDK assumes it is being run from resource URIs
       let chromeReg = Cc["@mozilla.org/chrome/chrome-registry;1"].getService(Ci.nsIChromeRegistry);
       let realPath = chromeReg.convertChromeURL(Services.io.newURI(TEST_PACKAGE, null, null));
       let resProtocol = Cc["@mozilla.org/network/protocol;1?name=resource"].getService(Ci.nsIResProtocolHandler);
       resProtocol.setSubstitution("jetpack-package-tests", realPath);
 
       // Set the test options
       const options = {
--- a/testing/mochitest/manifestLibrary.js
+++ b/testing/mochitest/manifestLibrary.js
@@ -56,16 +56,17 @@ function getTestManifest(url, params, ca
         callback({});
       }
     }
   }
   req.send();
 }
 
 // Test Filtering Code
+// TODO Only used by ipc tests, remove once those are implemented sanely
 
 /*
  Open the file referenced by runOnly|exclude and use that to compare against
  testList
  parameters:
    filter = json object of runtests | excludetests
    testList = array of test names to run
    runOnly = use runtests vs excludetests in case both are defined
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -41,27 +41,28 @@ from automationutils import (
     ShutdownLeaks,
     printstatus,
     LSANLeaks,
     setAutomationLog,
 )
 
 from datetime import datetime
 from manifestparser import TestManifest
-from manifestparser.filters import subsuite
+from manifestparser.filters import (
+    chunk_by_dir,
+    chunk_by_slice,
+    subsuite,
+)
 from mochitest_options import MochitestOptions
 from mozprofile import Profile, Preferences
 from mozprofile.permissions import ServerLocations
 from urllib import quote_plus as encodeURIComponent
 from mozlog.structured.formatters import TbplFormatter
 from mozlog.structured import commandline
 
-# This should use the `which` module already in tree, but it is
-# not yet present in the mozharness environment
-from mozrunner.utils import findInPath as which
 
 ###########################
 # Option for NSPR logging #
 ###########################
 
 # Set the desired log modules you want an NSPR log be produced by a try run for, or leave blank to disable the feature.
 # This will be passed to NSPR_LOG_MODULES environment variable. Try run will then put a download link for the log file
 # on tbpl.mozilla.org.
@@ -559,18 +560,16 @@ class MochitestUtilsMixin(object):
         """ Add test control options from the command line to the url
 
             URL parameters to test URL:
 
             autorun -- kick off tests automatically
             closeWhenDone -- closes the browser after the tests
             hideResultsTable -- hides the table of individual test results
             logFile -- logs test run to an absolute path
-            totalChunks -- how many chunks to split tests into
-            thisChunk -- which chunk to run
             startAt -- name of test to start at
             endAt -- name of test to end at
             timeout -- per-test timeout in seconds
             repeat -- How many times to repeat the test, ie: repeat=1 will run the test twice.
         """
 
         if not hasattr(options, 'logFile'):
             options.logFile = ""
@@ -607,21 +606,16 @@ class MochitestUtilsMixin(object):
                     "fileLevel=" +
                     encodeURIComponent(
                         options.fileLevel))
             if options.consoleLevel:
                 self.urlOpts.append(
                     "consoleLevel=" +
                     encodeURIComponent(
                         options.consoleLevel))
-            if options.totalChunks:
-                self.urlOpts.append("totalChunks=%d" % options.totalChunks)
-                self.urlOpts.append("thisChunk=%d" % options.thisChunk)
-            if options.chunkByDir:
-                self.urlOpts.append("chunkByDir=%d" % options.chunkByDir)
             if options.startAt:
                 self.urlOpts.append("startAt=%s" % options.startAt)
             if options.endAt:
                 self.urlOpts.append("endAt=%s" % options.endAt)
             if options.shuffle:
                 self.urlOpts.append("shuffle=1")
             if "MOZ_HIDE_RESULTS_TABLE" in env and env[
                     "MOZ_HIDE_RESULTS_TABLE"] == "1":
@@ -633,22 +627,16 @@ class MochitestUtilsMixin(object):
             if os.path.isfile(
                 os.path.join(
                     self.oldcwd,
                     os.path.dirname(__file__),
                     self.TEST_PATH,
                     options.testPath)) and options.repeat > 0:
                 self.urlOpts.append("testname=%s" %
                                     ("/").join([self.TEST_PATH, options.testPath]))
-            if options.testManifest:
-                self.urlOpts.append("testManifest=%s" % options.testManifest)
-                if hasattr(options, 'runOnly') and options.runOnly:
-                    self.urlOpts.append("runOnly=true")
-                else:
-                    self.urlOpts.append("runOnly=false")
             if options.manifestFile:
                 self.urlOpts.append("manifestFile=%s" % options.manifestFile)
             if options.failureFile:
                 self.urlOpts.append(
                     "failureFile=%s" %
                     self.getFullPath(
                         options.failureFile))
             if options.runSlower:
@@ -1858,45 +1846,89 @@ class Mochitest(MochitestUtilsMixin):
 
     def getActiveTests(self, options, disabled=True):
         """
           This method is used to parse the manifest and return active filtered tests.
         """
         self.setTestRoot(options)
         manifest = self.getTestManifest(options)
         if manifest:
-            # Python 2.6 doesn't allow unicode keys to be used for keyword
-            # arguments. This gross hack works around the problem until we
-            # rid ourselves of 2.6.
-            info = {}
-            for k, v in mozinfo.info.items():
-                if isinstance(k, unicode):
-                    k = k.encode('ascii')
-                info[k] = v
+            info = mozinfo.info
 
             # Bug 883858 - return all tests including disabled tests
             testPath = self.getTestPath(options)
             testPath = testPath.replace('\\', '/')
             if testPath.endswith('.html') or \
                testPath.endswith('.xhtml') or \
                testPath.endswith('.xul') or \
                testPath.endswith('.js'):
-                    # In the case where we have a single file, we don't want to
-                    # filter based on options such as subsuite.
-                tests = manifest.active_tests(disabled=disabled, **info)
+                # In the case where we have a single file, we don't want to
+                # filter based on options such as subsuite.
+                tests = manifest.active_tests(
+                    exists=False, disabled=disabled, **info)
                 for test in tests:
                     if 'disabled' in test:
                         del test['disabled']
 
             else:
-                filters = [subsuite(options.subsuite)]
+                # Bug 1089034 - imptest failure expectations are encoded as
+                # test manifests, even though they aren't tests. This gross
+                # hack causes several problems in automation including
+                # throwing off the chunking numbers. Remove them manually
+                # until bug 1089034 is fixed.
+                def remove_imptest_failure_expectations(tests, values):
+                    return (t for t in tests
+                            if 'imptests/failures' not in t['path'])
+
+                # filter that implements old-style JSON manifests, remove
+                # once everything is using .ini
+                def apply_json_manifest(tests, values):
+                    m = os.path.join(SCRIPT_DIR, options.testManifest)
+                    with open(m, 'r') as f:
+                        m = json.loads(f.read())
+
+                    runtests = m.get('runtests')
+                    exctests = m.get('excludetests')
+                    if runtests is None and exctests is None:
+                        if options.runOnly:
+                            runtests = m
+                        else:
+                            exctests = m
+
+                    disabled = 'disabled by {}'.format(options.testManifest)
+                    for t in tests:
+                        if runtests and not any(t['relpath'].startswith(r)
+                                                for r in runtests):
+                            t['disabled'] = disabled
+                        if exctests and any(t['relpath'].startswith(r)
+                                            for r in exctests):
+                            t['disabled'] = disabled
+                        yield t
+
+                filters = [
+                    remove_imptest_failure_expectations,
+                    subsuite(options.subsuite),
+                ]
+
+                if options.testManifest:
+                    filters.append(apply_json_manifest)
+
+                # Add chunking filters if specified
+                if options.chunkByDir:
+                    filters.append(chunk_by_dir(options.thisChunk,
+                                                options.totalChunks,
+                                                options.chunkByDir))
+                elif options.totalChunks:
+                    filters.append(chunk_by_slice(options.thisChunk,
+                                                  options.totalChunks))
                 tests = manifest.active_tests(
-                    disabled=disabled, filters=filters, **info)
+                    exists=False, disabled=disabled, filters=filters, **info)
                 if len(tests) == 0:
-                    tests = manifest.active_tests(disabled=True, **info)
+                    tests = manifest.active_tests(
+                        exists=False, disabled=True, **info)
 
         paths = []
 
         for test in tests:
             if len(tests) == 1 and 'disabled' in test:
                 del test['disabled']
 
             pathAbs = os.path.abspath(test['path'])
@@ -2006,25 +2038,16 @@ class Mochitest(MochitestUtilsMixin):
             options.runByDir = True
 
         if not options.runByDir:
             return self.runMochitests(options, onLaunch)
 
         # code for --run-by-dir
         dirs = self.getDirectories(options)
 
-        if options.totalChunks > 1:
-            chunkSize = int(len(dirs) / options.totalChunks) + 1
-            start = chunkSize * (options.thisChunk - 1)
-            end = chunkSize * (options.thisChunk)
-            dirs = dirs[start:end]
-
-        options.totalChunks = None
-        options.thisChunk = None
-        options.chunkByDir = 0
         result = 1  # default value, if no tests are run.
         inputTestPath = self.getTestPath(options)
         for dir in dirs:
             if inputTestPath and not inputTestPath.startswith(dir):
                 continue
 
             options.testPath = dir
             print "testpath: %s" % options.testPath
--- a/testing/mochitest/runtestsremote.py
+++ b/testing/mochitest/runtestsremote.py
@@ -1,37 +1,36 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import base64
 import json
 import logging
-import math
 import os
-import re
 import shutil
 import sys
 import tempfile
 import traceback
 
 sys.path.insert(
     0, os.path.abspath(
         os.path.realpath(
             os.path.dirname(__file__))))
 
 from automation import Automation
 from remoteautomation import RemoteAutomation, fennecLogcatFilters
 from runtests import Mochitest, MessageLogger
 from mochitest_options import MochitestOptions
 from mozlog import structured
 
+from manifestparser import TestManifest
+from manifestparser.filters import chunk_by_slice
 import devicemanager
 import droid
-import manifestparser
 import mozinfo
 import moznetwork
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 
 
 class RemoteOptions(MochitestOptions):
 
@@ -794,34 +793,25 @@ def main(args):
     dm.killProcess(procName)
 
     if options.robocopIni != "":
         # turning buffering off as it's not used in robocop
         message_logger.buffering = False
 
         # sut may wait up to 300 s for a robocop am process before returning
         dm.default_timeout = 320
-        mp = manifestparser.TestManifest(strict=False)
+        mp = TestManifest(strict=False)
         # TODO: pull this in dynamically
         mp.read(options.robocopIni)
-        robocop_tests = mp.active_tests(exists=False, **mozinfo.info)
-        tests = []
-        my_tests = tests
-        for test in robocop_tests:
-            tests.append(test['name'])
 
+        filters = []
         if options.totalChunks:
-            tests_per_chunk = math.ceil(
-                len(tests) / (options.totalChunks * 1.0))
-            start = int(round((options.thisChunk - 1) * tests_per_chunk))
-            end = int(round(options.thisChunk * tests_per_chunk))
-            if end > len(tests):
-                end = len(tests)
-            my_tests = tests[start:end]
-            log.info("Running tests %d-%d/%d" % (start + 1, end, len(tests)))
+            filters.append(
+                chunk_by_slice(options.thisChunk, options.totalChunks))
+        robocop_tests = mp.active_tests(exists=False, filters=filters, **mozinfo.info)
 
         options.extraPrefs.append('browser.search.suggest.enabled=true')
         options.extraPrefs.append('browser.search.suggest.prompted=true')
         options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
         options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
         options.extraPrefs.append('browser.snippets.enabled=false')
         options.extraPrefs.append('browser.casting.enabled=true')
 
@@ -830,19 +820,16 @@ def main(args):
 
         retVal = None
         # Filtering tests
         active_tests = []
         for test in robocop_tests:
             if options.testPath and options.testPath != test['name']:
                 continue
 
-            if not test['name'] in my_tests:
-                continue
-
             if 'disabled' in test:
                 log.info(
                     'TEST-INFO | skipping %s | %s' %
                     (test['name'], test['disabled']))
                 continue
 
             active_tests.append(test)
 
--- a/testing/mochitest/tests/SimpleTest/setup.js
+++ b/testing/mochitest/tests/SimpleTest/setup.js
@@ -155,35 +155,32 @@ TestRunner.logger.addListener("dumpListe
   dump(msg.info.join(' ') + "\n");
 });
 
 var gTestList = [];
 var RunSet = {};
 RunSet.runall = function(e) {
   // Filter tests to include|exclude tests based on data in params.filter.
   // This allows for including or excluding tests from the gTestList
+  // TODO Only used by ipc tests, remove once those are implemented sanely
   if (params.testManifest) {
     getTestManifest("http://mochi.test:8888/" + params.testManifest, params, function(filter) { gTestList = filterTests(filter, gTestList, params.runOnly); RunSet.runtests(); });
   } else {
     RunSet.runtests();
   }
 }
 
 RunSet.runtests = function(e) {
   // Which tests we're going to run
   var my_tests = gTestList;
 
   if (params.startAt || params.endAt) {
     my_tests = skipTests(my_tests, params.startAt, params.endAt);
   }
 
-  if (params.totalChunks && params.thisChunk) {
-    my_tests = chunkifyTests(my_tests, params.totalChunks, params.thisChunk, params.chunkByDir, TestRunner.logger);
-  }
-
   if (params.shuffle) {
     for (var i = my_tests.length-1; i > 0; --i) {
       var j = Math.floor(Math.random() * i);
       var tmp = my_tests[j];
       my_tests[j] = my_tests[i];
       my_tests[i] = tmp;
     }
   }
--- a/testing/mozbase/manifestparser/manifestparser/filters.py
+++ b/testing/mozbase/manifestparser/manifestparser/filters.py
@@ -4,16 +4,17 @@
 
 """
 A filter is a callable that accepts an iterable of test objects and a
 dictionary of values, and returns a new iterable of test objects. It is
 possible to define custom filters if the built-in ones are not enough.
 """
 
 from collections import defaultdict, MutableSequence
+import itertools
 import os
 
 from .expression import (
     parse,
     ParseError,
 )
 
 
@@ -207,28 +208,39 @@ class chunk_by_dir(InstanceFilter):
 
             if path.startswith(os.sep):
                 path = path[1:]
 
             dirs = path.split(os.sep)
             dirs = dirs[:min(self.depth, len(dirs)-1)]
             path = os.sep.join(dirs)
 
-            if path not in tests_by_dir:
+            # don't count directories that only have disabled tests in them,
+            # but still yield disabled tests that are alongside enabled tests
+            if path not in ordered_dirs and 'disabled' not in test:
                 ordered_dirs.append(path)
             tests_by_dir[path].append(test)
 
-        tests_per_chunk = float(len(tests_by_dir)) / self.total_chunks
+        tests_per_chunk = float(len(ordered_dirs)) / self.total_chunks
         start = int(round((self.this_chunk - 1) * tests_per_chunk))
         end = int(round(self.this_chunk * tests_per_chunk))
 
         for i in range(start, end):
-            for test in tests_by_dir[ordered_dirs[i]]:
+            for test in tests_by_dir.pop(ordered_dirs[i]):
                 yield test
 
+        # find directories that only contain disabled tests. They still need to
+        # be yielded for reporting purposes. Put them all in chunk 1 for
+        # simplicity.
+        if self.this_chunk == 1:
+            disabled_dirs = [v for k, v in tests_by_dir.iteritems()
+                             if k not in ordered_dirs]
+            for disabled_test in itertools.chain(*disabled_dirs):
+                yield disabled_test
+
 
 class chunk_by_runtime(InstanceFilter):
     """
     Chunking algorithm that attempts to group tests into chunks based on their
     average runtimes. It keeps manifests of tests together and pairs slow
     running manifests with fast ones.
 
     :param this_chunk: the current chunk, 1 <= this_chunk <= total_chunks