Backed out changeset ea625e85c72a (bug 1123763) for checktest orange on a CLOSED TREE
authorWes Kocher <wkocher@mozilla.com>
Mon, 09 Feb 2015 14:00:13 -0800
changeset 228215 bc3240e8a37dd1932a32fab4fef32d66c2548db4
parent 228214 3f53cf53a34eb7a69a0d9456740d44f3b68d882a
child 228216 612eff467e8e16efac0bf6f67a7b31517d1e8322
push id55349
push userkwierso@gmail.com
push dateMon, 09 Feb 2015 22:00:28 +0000
treeherdermozilla-inbound@bc3240e8a37d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1123763
milestone38.0a1
backs outea625e85c72ad637bc24b71a647dde5aa086922d
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset ea625e85c72a (bug 1123763) for checktest orange on a CLOSED TREE
testing/mochitest/mochitest_options.py
testing/mochitest/runtests.py
testing/mozbase/docs/conf.py
testing/mozbase/docs/manifestparser.rst
testing/mozbase/manifestparser/manifestparser/filters.py
testing/mozbase/manifestparser/manifestparser/manifestparser.py
testing/mozbase/manifestparser/setup.py
testing/mozbase/manifestparser/tests/manifest.ini
testing/mozbase/manifestparser/tests/test_filters.py
testing/mozbase/manifestparser/tests/test_testmanifest.py
--- a/testing/mochitest/mochitest_options.py
+++ b/testing/mochitest/mochitest_options.py
@@ -158,17 +158,17 @@ class MochitestOptions(optparse.OptionPa
           "dest": "browserChrome",
           "help": "run browser chrome Mochitests",
           "default": False,
         }],
         [["--subsuite"],
         { "action": "store",
           "dest": "subsuite",
           "help": "subsuite of tests to run",
-          "default": None,
+          "default": "",
         }],
         [["--jetpack-package"],
         { "action": "store_true",
           "dest": "jetpackPackage",
           "help": "run jetpack package tests",
           "default": False,
         }],
         [["--jetpack-addon"],
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -41,17 +41,16 @@ from automationutils import (
     ShutdownLeaks,
     printstatus,
     LSANLeaks,
     setAutomationLog,
 )
 
 from datetime import datetime
 from manifestparser import TestManifest
-from manifestparser.filters import subsuite
 from mochitest_options import MochitestOptions
 from mozprofile import Profile, Preferences
 from mozprofile.permissions import ServerLocations
 from urllib import quote_plus as encodeURIComponent
 from mozlog.structured.formatters import TbplFormatter
 from mozlog.structured import commandline
 
 # This should use the `which` module already in tree, but it is
@@ -1644,27 +1643,25 @@ class Mochitest(MochitestUtilsMixin):
       # Bug 883858 - return all tests including disabled tests
       testPath = self.getTestPath(options)
       testPath = testPath.replace('\\', '/')
       if testPath.endswith('.html') or \
          testPath.endswith('.xhtml') or \
          testPath.endswith('.xul') or \
          testPath.endswith('.js'):
           # In the case where we have a single file, we don't want to filter based on options such as subsuite.
-          tests = manifest.active_tests(disabled=disabled, **info)
+          tests = manifest.active_tests(disabled=disabled, options=None, **info)
           for test in tests:
             if 'disabled' in test:
               del test['disabled']
 
       else:
-        filters = [subsuite(options.subsuite)]
-        tests = manifest.active_tests(
-            disabled=disabled, filters=filters, **info)
+        tests = manifest.active_tests(disabled=disabled, options=options, **info)
         if len(tests) == 0:
-          tests = manifest.active_tests(disabled=True, **info)
+          tests = manifest.active_tests(disabled=True, options=options, **info)
 
     paths = []
 
     for test in tests:
       if len(tests) == 1 and 'disabled' in test:
         del test['disabled']
 
       pathAbs = os.path.abspath(test['path'])
--- a/testing/mozbase/docs/conf.py
+++ b/testing/mozbase/docs/conf.py
@@ -93,25 +93,16 @@ pygments_style = 'sphinx'
 #modindex_common_prefix = []
 
 
 # -- Options for HTML output ---------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 html_theme = 'default'
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-
-if not on_rtd:
-    try:
-        import sphinx_rtd_theme
-        html_theme = 'sphinx_rtd_theme'
-        html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-    except ImportError:
-        pass
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
 #html_theme_path = []
--- a/testing/mozbase/docs/manifestparser.rst
+++ b/testing/mozbase/docs/manifestparser.rst
@@ -1,13 +1,11 @@
 Managing lists of tests
 =======================
 
-.. py:currentmodule:: manifestparser
-
 We don't always want to run all tests, all the time. Sometimes a test
 may be broken, in other cases we only want to run a test on a specific
 platform or build of Mozilla. To handle these cases (and more), we
 created a python library to create and use test "manifests", which
 codify this information.
 
 :mod:`manifestparser` --- Create and manage test manifests
 -----------------------------------------------------------
@@ -259,83 +257,43 @@ 3. Optionally, a harness will have an in
 from TestManifest if more harness-specific customization is desired at
 the manifest level.
 
 See the source code at https://github.com/mozilla/mozbase/tree/master/manifestparser
 and
 https://github.com/mozilla/mozbase/blob/master/manifestparser/manifestparser.py
 in particular.
 
-Filtering Manifests
-```````````````````
-
-After creating a `TestManifest` object, all manifest files are read and a list
-of test objects can be accessed via `TestManifest.tests`. However this list contains
-all test objects, whether they should be run or not. Normally they need to be
-filtered down only to the set of tests that should be run by the test harness.
-
-To do this, a test harness can call `TestManifest.active_tests`:
-
-.. code-block:: python
-
-    tests = manifest.active_tests(exists=True, disabled=True, **tags)
-
-By default, `active_tests` runs the filters found in
-:attr:`~.DEFAULT_FILTERS`. It also accepts two convenience arguments:
-
-1. `exists`: if True (default), filter out tests that do not exist on the local file system.
-2. `disabled`: if True (default), do not filter out tests containing the 'disabled' key
-   (which can be set by `skip-if`, `run-if` or manually).
+Using Manifests
+```````````````
 
-This works for simple cases, but there are other built-in filters, or even custom filters
-that can be applied to the `TestManifest`. To do so, add the filter to `TestManifest.filters`:
-
-.. code-block:: python
-
-    from manifestparser.filters import subsuite
-    import mozinfo
-
-    filters = [subsuite('devtools')]
-    tests = manifest.active_tests(filters=filters, **mozinfo.info)
-
-.. automodule:: manifestparser.filters
-    :members:
-    :exclude-members: filterlist,InstanceFilter,DEFAULT_FILTERS
-
-.. autodata::  manifestparser.filters.DEFAULT_FILTERS
-    :annotation:
-
-For example, suppose we want to introduce a new key called `timeout-if` that adds a
-'timeout' property to a test if a certain condition is True. The syntax in the manifest
-files will look like this:
+A test harness will normally call `TestManifest.active_tests`:
 
 .. code-block:: text
 
-    [test_foo.py]
-    timeout-if = 300, os == 'win'
+    def active_tests(self, exists=True, disabled=True, **tags):
 
-The value is <timeout>, <condition> where condition is the same format as the one in
-`skip-if`. In the above case, if os == 'win', a timeout of 300 seconds will be
-applied. Otherwise, no timeout will be applied. All we need to do is define the filter
-and add it:
-
-.. code-block:: python
+The manifests are passed to the `__init__` or `read` methods with
+appropriate arguments.  `active_tests` then allows you to select the
+tests you want:
 
-    from manifestparser.expression import parse
-    import mozinfo
+- exists : return only existing tests
+- disabled : whether to return disabled tests; if not these will be
+  filtered out; if True (the default), the `disabled` key of a
+  test's metadata will be present and will be set to the reason that a
+  test is disabled
+- tags : keys and values to filter on (e.g. `os='linux'`)
 
-    def timeout_if(tests, values):
-        for test in tests:
-            if 'timeout-if' in test:
-                timeout, condition = test['timeout-if'].split(',', 1)
-                if parse(condition, **values):
-                    test['timeout'] = timeout
-            yield test
-
-    tests = manifest.active_tests(filters=[timeout_if], **mozinfo.info)
+`active_tests` looks for tests with `skip-if`
+`run-if`.  If the condition is or is not fulfilled,
+respectively, the test is marked as disabled.  For instance, if you
+pass `**dict(os='linux')` as `**tags`, if a test contains a line
+`skip-if = os == 'linux'` this test will be disabled, or
+`run-if = os = 'win'` in which case the test will also be disabled.  It
+is up to the harness to pass in tags appropriate to its usage.
 
 Creating Manifests
 ``````````````````
 
 manifestparser comes with a console script, `manifestparser create`, that
 may be used to create a seed manifest structure from a directory of
 files.  Run `manifestparser help create` for usage information.
 
deleted file mode 100644
--- a/testing/mozbase/manifestparser/manifestparser/filters.py
+++ /dev/null
@@ -1,179 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-"""
-A filter is a callable that accepts an iterable of test objects and a
-dictionary of values, and returns a new iterable of test objects. It is
-possible to define custom filters if the built-in ones are not enough.
-"""
-
-from collections import MutableSequence
-import os
-
-from .expression import (
-    parse,
-    ParseError,
-)
-
-
-# built-in filters
-
-def skip_if(tests, values):
-    """
-    Sets disabled on all tests containing the `skip-if` tag and whose condition
-    is True. This filter is added by default.
-    """
-    tag = 'skip-if'
-    for test in tests:
-        if tag in test and parse(test[tag], **values):
-            test.setdefault('disabled', '{}: {}'.format(tag, test[tag]))
-        yield test
-
-
-def run_if(tests, values):
-    """
-    Sets disabled on all tests containing the `run-if` tag and whose condition
-    is False. This filter is added by default.
-    """
-    tag = 'run-if'
-    for test in tests:
-        if tag in test and not parse(test[tag], **values):
-            test.setdefault('disabled', '{}: {}'.format(tag, test[tag]))
-        yield test
-
-
-def fail_if(tests, values):
-    """
-    Sets expected to 'fail' on all tests containing the `fail-if` tag and whose
-    condition is True. This filter is added by default.
-    """
-    tag = 'fail-if'
-    for test in tests:
-        if tag in test and parse(test[tag], **values):
-            test['expected'] = 'fail'
-        yield test
-
-
-def enabled(tests, values):
-    """
-    Removes all tests containing the `disabled` key. This filter can be
-    added by passing `disabled=False` into `active_tests`.
-    """
-    for test in tests:
-        if 'disabled' not in test:
-            yield test
-
-
-def exists(tests, values):
-    """
-    Removes all tests that do not exist on the file system. This filter is
-    added by default, but can be removed by passing `exists=False` into
-    `active_tests`.
-    """
-    for test in tests:
-        if os.path.exists(test['path']):
-            yield test
-
-
-# built-in instance filters
-
-class InstanceFilter(object):
-    """
-    Generally only one instance of a class filter should be applied at a time.
-    Two instances of `InstanceFilter` are considered equal if they have the
-    same class name. This ensures only a single instance is ever added to
-    `filterlist`.
-    """
-    def __eq__(self, other):
-        return self.__class__ == other.__class__
-
-
-class subsuite(InstanceFilter):
-    """
-    If `name` is None, removes all tests that have a `subsuite` key.
-    Otherwise removes all tests that do not have a subsuite matching `name`.
-
-    It is possible to specify conditional subsuite keys using:
-       subsuite = foo,condition
-
-    where 'foo' is the subsuite name, and 'condition' is the same type of
-    condition used for skip-if.  If the condition doesn't evaluate to true,
-    the subsuite designation will be removed from the test.
-
-    :param name: The name of the subsuite to run (default None)
-    """
-    def __init__(self, name=None):
-        self.name = name
-
-    def __call__(self, tests, values):
-        # Look for conditional subsuites, and replace them with the subsuite
-        # itself (if the condition is true), or nothing.
-        for test in tests:
-            subsuite = test.get('subsuite', '')
-            if ',' in subsuite:
-                try:
-                    subsuite, cond = subsuite.split(',')
-                except ValueError:
-                    raise ParseError("subsuite condition can't contain commas")
-                matched = parse(cond, **values)
-                if matched:
-                    test['subsuite'] = subsuite
-                else:
-                    test['subsuite'] = ''
-
-            # Filter on current subsuite
-            if self.name is None:
-                if not test.get('subsuite'):
-                    yield test
-            else:
-                if test.get('subsuite') == self.name:
-                    yield test
-
-
-# filter container
-
-DEFAULT_FILTERS = (
-    skip_if,
-    run_if,
-    fail_if,
-)
-"""
-By default :func:`~.active_tests` will run the :func:`~.skip_if`,
-:func:`~.run_if` and :func:`~.fail_if` filters.
-"""
-
-
-class filterlist(MutableSequence):
-    """
-    A MutableSequence that raises TypeError when adding a non-callable and
-    ValueError if the item is already added.
-    """
-
-    def __init__(self, items=None):
-        self.items = []
-        if items:
-            self.items = list(items)
-
-    def _validate(self, item):
-        if not callable(item):
-            raise TypeError("Filters must be callable!")
-        if item in self:
-            raise ValueError("Filter {} is already applied!".format(item))
-
-    def __getitem__(self, key):
-        return self.items[key]
-
-    def __setitem__(self, key, value):
-        self._validate(value)
-        self.items[key] = value
-
-    def __delitem__(self, key):
-        del self.items[key]
-
-    def __len__(self):
-        return len(self.items)
-
-    def insert(self, index, value):
-        self._validate(value)
-        self.items.insert(index, value)
old mode 100644
new mode 100755
--- a/testing/mozbase/manifestparser/manifestparser/manifestparser.py
+++ b/testing/mozbase/manifestparser/manifestparser/manifestparser.py
@@ -7,21 +7,19 @@
 from StringIO import StringIO
 import json
 import fnmatch
 import os
 import shutil
 import sys
 
 from .ini import read_ini
-from .filters import (
-    DEFAULT_FILTERS,
-    enabled,
-    exists as _exists,
-    filterlist,
+from .expression import (
+    parse,
+    ParseError,
 )
 
 relpath = os.path.relpath
 string = (basestring,)
 
 
 ### path normalization
 
@@ -308,23 +306,21 @@ class ManifestParser(object):
 
     def paths(self):
         return [i['path'] for i in self.tests]
 
 
     ### methods for auditing
 
     def missing(self, tests=None):
-        """
-        return list of tests that do not exist on the filesystem
-        """
+        """return list of tests that do not exist on the filesystem"""
         if tests is None:
             tests = self.tests
-        existing = _exists(tests, {})
-        return [t for t in tests if t not in existing]
+        return [test for test in tests
+                if not os.path.exists(test['path'])]
 
     def check_missing(self, tests=None):
         missing = self.missing(tests=tests)
         if missing:
             missing_paths = [test['path'] for test in missing]
             if self.strict:
                 raise IOError("Strict mode enabled, test paths must exist. "
                               "The following test(s) are missing: %s" %
@@ -702,48 +698,120 @@ convert = ManifestParser.from_directorie
 
 
 class TestManifest(ManifestParser):
     """
     apply logic to manifests;  this is your integration layer :)
     specific harnesses may subclass from this if they need more logic
     """
 
-    def __init__(self, *args, **kwargs):
-        ManifestParser.__init__(self, *args, **kwargs)
-        self.filters = filterlist(DEFAULT_FILTERS)
-
-    def active_tests(self, exists=True, disabled=True, filters=None, **values):
+    def filter(self, values, tests):
+        """
+        filter on a specific list tag, e.g.:
+        run-if = os == win linux
+        skip-if = os == mac
         """
-        Run all applied filters on the set of tests.
+
+        # tags:
+        run_tag = 'run-if'
+        skip_tag = 'skip-if'
+        fail_tag = 'fail-if'
+
+        cache = {}
+
+        def _parse(cond):
+            if '#' in cond:
+                cond = cond[:cond.index('#')]
+            cond = cond.strip()
+            if cond in cache:
+                ret = cache[cond]
+            else:
+                ret = parse(cond, **values)
+                cache[cond] = ret
+            return ret
+
+        # loop over test
+        for test in tests:
+            reason = None # reason to disable
 
-        :param exists: filter out non-existing tests (default True)
-        :param disabled: whether to return disabled tests (default True)
-        :param values: keys and values to filter on (e.g. `os = linux mac`)
-        :param filters: list of filters to apply to the tests
-        :returns: list of test objects that were not filtered out
+            # tagged-values to run
+            if run_tag in test:
+                condition = test[run_tag]
+                if not _parse(condition):
+                    reason = '%s: %s' % (run_tag, condition)
+
+            # tagged-values to skip
+            if skip_tag in test:
+                condition = test[skip_tag]
+                if _parse(condition):
+                    reason = '%s: %s' % (skip_tag, condition)
+
+            # mark test as disabled if there's a reason
+            if reason:
+                test.setdefault('disabled', reason)
+
+            # mark test as a fail if so indicated
+            if fail_tag in test:
+                condition = test[fail_tag]
+                if _parse(condition):
+                    test['expected'] = 'fail'
+
+    def active_tests(self, exists=True, disabled=True, options=None, **values):
+        """
+        - exists : return only existing tests
+        - disabled : whether to return disabled tests
+        - options: an optparse or argparse options object, used for subsuites
+        - values : keys and values to filter on (e.g. `os = linux mac`)
         """
         tests = [i.copy() for i in self.tests] # shallow copy
 
-        # mark all tests as passing
+        # Conditional subsuites are specified using:
+        #    subsuite = foo,condition
+        # where 'foo' is the subsuite name, and 'condition' is the same type of
+        # condition used for skip-if.  If the condition doesn't evaluate to true,
+        # the subsuite designation will be removed from the test.
+        #
+        # Look for conditional subsuites, and replace them with the subsuite itself
+        # (if the condition is true), or nothing.
+        for test in tests:
+            subsuite = test.get('subsuite', '')
+            if ',' in subsuite:
+                try:
+                    subsuite, condition = subsuite.split(',')
+                except ValueError:
+                    raise ParseError("subsuite condition can't contain commas")
+                # strip any comments from the condition
+                condition = condition.split('#')[0]
+                matched = parse(condition, **values)
+                if matched:
+                    test['subsuite'] = subsuite
+                else:
+                    test['subsuite'] = ''
+
+        # Filter on current subsuite
+        if options:
+            if hasattr(options, 'subsuite') and options.subsuite:
+                tests = [test for test in tests if options.subsuite == test['subsuite']]
+            else:
+                tests = [test for test in tests if not test['subsuite']]
+
+        # mark all tests as passing unless indicated otherwise
         for test in tests:
             test['expected'] = test.get('expected', 'pass')
 
-        # make a copy so original doesn't get modified
-        fltrs = self.filters[:]
+        # ignore tests that do not exist
         if exists:
-            if self.strict:
-                self.check_missing(tests)
-            else:
-                fltrs.append(_exists)
+            missing = self.check_missing(tests)
+            tests = [test for test in tests if test not in missing]
 
+        # filter by tags
+        self.filter(values, tests)
+
+        # ignore disabled tests if specified
         if not disabled:
-            fltrs.append(enabled)
+            tests = [test for test in tests
+                     if not 'disabled' in test]
 
-        if filters:
-            fltrs += filters
-
-        for fn in fltrs:
-            tests = fn(tests, values)
-        return list(tests)
+        # return active tests
+        return tests
 
     def test_paths(self):
         return [test['path'] for test in self.active_tests()]
--- a/testing/mozbase/manifestparser/setup.py
+++ b/testing/mozbase/manifestparser/setup.py
@@ -1,16 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from setuptools import setup
 
 PACKAGE_NAME = "manifestparser"
-PACKAGE_VERSION = '1.0'
+PACKAGE_VERSION = '0.9'
 
 setup(name=PACKAGE_NAME,
       version=PACKAGE_VERSION,
       description="Library to create and manage test manifests",
       long_description="see http://mozbase.readthedocs.org/",
       classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
       keywords='mozilla manifests',
       author='Mozilla Automation and Testing Team',
--- a/testing/mozbase/manifestparser/tests/manifest.ini
+++ b/testing/mozbase/manifestparser/tests/manifest.ini
@@ -1,11 +1,9 @@
 # test manifest for manifestparser
 [test_expressionparser.py]
 [test_manifestparser.py]
 [test_testmanifest.py]
 [test_read_ini.py]
 [test_convert_directory.py]
-[test_filters.py]
-[test_chunking.py]
 
 [test_convert_symlinks.py]
 disabled = https://bugzilla.mozilla.org/show_bug.cgi?id=920938
deleted file mode 100644
--- a/testing/mozbase/manifestparser/tests/test_filters.py
+++ /dev/null
@@ -1,156 +0,0 @@
-#!/usr/bin/env python
-
-from copy import deepcopy
-import os
-import unittest
-
-from manifestparser import TestManifest
-from manifestparser.filters import (
-    subsuite,
-    skip_if,
-    run_if,
-    fail_if,
-    enabled,
-    exists,
-    filterlist,
-)
-
-here = os.path.dirname(os.path.abspath(__file__))
-
-
-class FilterList(unittest.TestCase):
-    """Test filterlist datatype"""
-
-    def test_data_model(self):
-        foo = lambda x, y: x
-        bar = lambda x, y: x
-        baz = lambda x, y: x
-        fl = filterlist()
-
-        fl.extend([foo, bar])
-        self.assertEquals(len(fl), 2)
-        self.assertTrue(foo in fl)
-
-        fl.append(baz)
-        self.assertEquals(fl[2], baz)
-
-        fl.remove(baz)
-        self.assertFalse(baz in fl)
-
-        item = fl.pop()
-        self.assertEquals(item, bar)
-
-        self.assertEquals(fl.index(foo), 0)
-
-        del fl[0]
-        self.assertFalse(foo in fl)
-        with self.assertRaises(IndexError):
-            fl[0]
-
-    def test_add_non_callable_to_set(self):
-        fl = filterlist()
-        with self.assertRaises(TypeError):
-            fl.append('foo')
-
-    def test_add_duplicates_to_set(self):
-        foo = lambda x, y: x
-        bar = lambda x, y: x
-        sub = subsuite('foo')
-        fl = filterlist([foo, bar, sub])
-        self.assertEquals(len(fl), 3)
-        self.assertEquals(fl[0], foo)
-
-        with self.assertRaises(ValueError):
-            fl.append(foo)
-
-        with self.assertRaises(ValueError):
-            fl.append(subsuite('bar'))
-
-    def test_filters_run_in_order(self):
-        a = lambda x, y: x
-        b = lambda x, y: x
-        c = lambda x, y: x
-        d = lambda x, y: x
-        e = lambda x, y: x
-        f = lambda x, y: x
-
-        fl = filterlist([a, b])
-        fl.append(c)
-        fl.extend([d, e])
-        fl += [f]
-        self.assertEquals([i for i in fl], [a, b, c, d, e, f])
-
-
-class BuiltinFilters(unittest.TestCase):
-    """Test the built-in filters"""
-
-    tests = (
-        { "name": "test0" },
-        { "name": "test1", "skip-if": "foo == 'bar'" },
-        { "name": "test2", "run-if": "foo == 'bar'" },
-        { "name": "test3", "fail-if": "foo == 'bar'" },
-        { "name": "test4", "disabled": "some reason" },
-        { "name": "test5", "subsuite": "baz" },
-        { "name": "test6", "subsuite": "baz,foo == 'bar'" })
-
-    def test_skip_if(self):
-        tests = deepcopy(self.tests)
-        tests = list(skip_if(tests, {}))
-        self.assertEquals(len(tests), len(self.tests))
-
-        tests = deepcopy(self.tests)
-        tests = list(skip_if(tests, {'foo': 'bar'}))
-        self.assertNotIn(self.tests[1], tests)
-
-    def test_run_if(self):
-        tests = deepcopy(self.tests)
-        tests = list(run_if(tests, {}))
-        self.assertNotIn(self.tests[2], tests)
-
-        tests = deepcopy(self.tests)
-        tests = list(run_if(tests, {'foo': 'bar'}))
-        self.assertEquals(len(tests), len(self.tests))
-
-    def test_fail_if(self):
-        tests = deepcopy(self.tests)
-        tests = list(fail_if(tests, {}))
-        self.assertNotIn('expected', tests[3])
-
-        tests = deepcopy(self.tests)
-        tests = list(fail_if(tests, {'foo': 'bar'}))
-        self.assertEquals(tests[3]['expected'], 'fail')
-
-    def test_enabled(self):
-        tests = deepcopy(self.tests)
-        tests = list(enabled(tests, {}))
-        self.assertNotIn(self.tests[4], tests)
-
-    def test_subsuite(self):
-        sub1 = subsuite()
-        sub2 = subsuite('baz')
-
-        tests = deepcopy(self.tests)
-        tests = list(sub1(tests, {}))
-        self.assertNotIn(self.tests[5], tests)
-        self.assertEquals(tests[-1]['name'], 'test6')
-
-        tests = deepcopy(self.tests)
-        tests = list(sub2(tests, {}))
-        self.assertEquals(len(tests), 1)
-        self.assertIn(self.tests[5], tests)
-
-    def test_subsuite_condition(self):
-        sub1 = subsuite()
-        sub2 = subsuite('baz')
-
-        tests = deepcopy(self.tests)
-
-        tests = list(sub1(tests, {'foo': 'bar'}))
-        self.assertNotIn(self.tests[5], tests)
-        self.assertNotIn(self.tests[6], tests)
-
-        tests = deepcopy(self.tests)
-        tests = list(sub2(tests, {'foo': 'bar'}))
-        self.assertEquals(len(tests), 2)
-        self.assertEquals(tests[0]['name'], 'test5')
-        self.assertEquals(tests[1]['name'], 'test6')
--- a/testing/mozbase/manifestparser/tests/test_testmanifest.py
+++ b/testing/mozbase/manifestparser/tests/test_testmanifest.py
@@ -1,17 +1,15 @@
 #!/usr/bin/env python
 
 import os
 import shutil
 import tempfile
 import unittest
-
 from manifestparser import TestManifest, ParseError
-from manifestparser.filters import subsuite
 
 here = os.path.dirname(os.path.abspath(__file__))
 
 
 class TestTestManifest(unittest.TestCase):
     """Test the Test Manifest"""
 
     def test_testmanifest(self):
@@ -23,21 +21,21 @@ class TestTestManifest(unittest.TestCase
         self.assertEqual([i['name'] for i in manifest.active_tests(os='linux', disabled=False, exists=False)],
                          ['fleem', 'linuxtest'])
 
         # Look for existing tests.  There is only one:
         self.assertEqual([i['name'] for i in manifest.active_tests()],
                          ['fleem'])
 
         # You should be able to expect failures:
-        last = manifest.active_tests(exists=False, toolkit='gtk2')[-1]
-        self.assertEqual(last['name'], 'linuxtest')
-        self.assertEqual(last['expected'], 'pass')
-        last = manifest.active_tests(exists=False, toolkit='cocoa')[-1]
-        self.assertEqual(last['expected'], 'fail')
+        last_test = manifest.active_tests(exists=False, toolkit='gtk2')[-1]
+        self.assertEqual(last_test['name'], 'linuxtest')
+        self.assertEqual(last_test['expected'], 'pass')
+        last_test = manifest.active_tests(exists=False, toolkit='cocoa')[-1]
+        self.assertEqual(last_test['expected'], 'fail')
 
     def test_missing_paths(self):
         """
         Test paths that don't exist raise an exception in strict mode.
         """
         tempdir = tempfile.mkdtemp()
 
         missing_path = os.path.join(here, 'missing-path.ini')
@@ -58,55 +56,57 @@ class TestTestManifest(unittest.TestCase
         self.assertEqual(len(manifest.tests), 8)
         names = [i['name'] for i in manifest.tests]
         self.assertFalse('test_0202_app_launch_apply_update_dirlocked.js' in names)
 
     def test_manifest_subsuites(self):
         """
         test subsuites and conditional subsuites
         """
+        class AttributeDict(dict):
+            def __getattr__(self, attr):
+                return self[attr]
+            def __setattr__(self, attr, value):
+                self[attr] = value
+
         relative_path = os.path.join(here, 'subsuite.ini')
         manifest = TestManifest(manifests=(relative_path,))
         info = {'foo': 'bar'}
+        options = {'subsuite': 'bar'}
 
         # 6 tests total
-        tests = manifest.active_tests(exists=False, **info)
-        self.assertEquals(len(tests), 6)
+        self.assertEquals(len(manifest.active_tests(exists=False, **info)), 6)
 
         # only 3 tests for subsuite bar when foo==bar
-        tests = manifest.active_tests(exists=False,
-                                      filters=[subsuite('bar')],
-                                      **info)
-        self.assertEquals(len(tests), 3)
+        self.assertEquals(len(manifest.active_tests(exists=False,
+                                                    options=AttributeDict(options),
+                                                    **info)), 3)
 
-        # only 1 test for subsuite baz, regardless of conditions
+        options = {'subsuite': 'baz'}
         other = {'something': 'else'}
-        tests = manifest.active_tests(exists=False,
-                                      filters=[subsuite('baz')],
-                                      **info)
-        self.assertEquals(len(tests), 1)
-        tests = manifest.active_tests(exists=False,
-                                      filters=[subsuite('baz')],
-                                      **other)
-        self.assertEquals(len(tests), 1)
+        # only 1 test for subsuite baz, regardless of conditions
+        self.assertEquals(len(manifest.active_tests(exists=False,
+                                                    options=AttributeDict(options),
+                                                    **info)), 1)
+        self.assertEquals(len(manifest.active_tests(exists=False,
+                                                    options=AttributeDict(options),
+                                                    **other)), 1)
 
         # 4 tests match when the condition doesn't match (all tests except
         # the unconditional subsuite)
         info = {'foo': 'blah'}
-        tests = manifest.active_tests(exists=False,
-                                      filters=[subsuite()],
-                                      **info)
-        self.assertEquals(len(tests), 5)
+        options = {'subsuite': None}
+        self.assertEquals(len(manifest.active_tests(exists=False,
+                                                    options=AttributeDict(options),
+                                                    **info)), 5)
 
         # test for illegal subsuite value
         manifest.tests[0]['subsuite'] = 'subsuite=bar,foo=="bar",type="nothing"'
-        with self.assertRaises(ParseError):
-            manifest.active_tests(exists=False,
-                                  filters=[subsuite('foo')],
-                                  **info)
+        self.assertRaises(ParseError, manifest.active_tests, exists=False,
+                          options=AttributeDict(options), **info)
 
     def test_none_and_empty_manifest(self):
         """
         Test TestManifest for None and empty manifest, see
         https://bugzilla.mozilla.org/show_bug.cgi?id=1087682
         """
         none_manifest = TestManifest(manifests=None, strict=False)
         self.assertEqual(len(none_manifest.test_paths()), 0)