Backed out changeset ea625e85c72a (bug 1123763) for checktest orange on a CLOSED TREE
authorWes Kocher <wkocher@mozilla.com>
Mon, 09 Feb 2015 14:00:13 -0800
changeset 241840 bc3240e8a37dd1932a32fab4fef32d66c2548db4
parent 241839 3f53cf53a34eb7a69a0d9456740d44f3b68d882a
child 241841 a7d06bafef8b360d04cdc4e6bd73213a8137d7b6
child 241853 612eff467e8e16efac0bf6f67a7b31517d1e8322
push id624
push userdburns@mozilla.com
push dateTue, 10 Feb 2015 13:30:25 +0000
bugs1123763
milestone38.0a1
backs outea625e85c72ad637bc24b71a647dde5aa086922d
Backed out changeset ea625e85c72a (bug 1123763) for checktest orange on a CLOSED TREE
testing/mochitest/mochitest_options.py
testing/mochitest/runtests.py
testing/mozbase/docs/conf.py
testing/mozbase/docs/manifestparser.rst
testing/mozbase/manifestparser/manifestparser/filters.py
testing/mozbase/manifestparser/manifestparser/manifestparser.py
testing/mozbase/manifestparser/setup.py
testing/mozbase/manifestparser/tests/manifest.ini
testing/mozbase/manifestparser/tests/test_filters.py
testing/mozbase/manifestparser/tests/test_testmanifest.py
--- a/testing/mochitest/mochitest_options.py
+++ b/testing/mochitest/mochitest_options.py
@@ -158,17 +158,17 @@ class MochitestOptions(optparse.OptionPa
           "dest": "browserChrome",
           "help": "run browser chrome Mochitests",
           "default": False,
         }],
         [["--subsuite"],
         { "action": "store",
           "dest": "subsuite",
           "help": "subsuite of tests to run",
-          "default": None,
+          "default": "",
         }],
         [["--jetpack-package"],
         { "action": "store_true",
           "dest": "jetpackPackage",
           "help": "run jetpack package tests",
           "default": False,
         }],
         [["--jetpack-addon"],
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -41,17 +41,16 @@ from automationutils import (
     ShutdownLeaks,
     printstatus,
     LSANLeaks,
     setAutomationLog,
 )
 
 from datetime import datetime
 from manifestparser import TestManifest
-from manifestparser.filters import subsuite
 from mochitest_options import MochitestOptions
 from mozprofile import Profile, Preferences
 from mozprofile.permissions import ServerLocations
 from urllib import quote_plus as encodeURIComponent
 from mozlog.structured.formatters import TbplFormatter
 from mozlog.structured import commandline
 
 # This should use the `which` module already in tree, but it is
@@ -1644,27 +1643,25 @@ class Mochitest(MochitestUtilsMixin):
       # Bug 883858 - return all tests including disabled tests
       testPath = self.getTestPath(options)
       testPath = testPath.replace('\\', '/')
       if testPath.endswith('.html') or \
          testPath.endswith('.xhtml') or \
          testPath.endswith('.xul') or \
          testPath.endswith('.js'):
           # In the case where we have a single file, we don't want to filter based on options such as subsuite.
-          tests = manifest.active_tests(disabled=disabled, **info)
+          tests = manifest.active_tests(disabled=disabled, options=None, **info)
           for test in tests:
             if 'disabled' in test:
               del test['disabled']
 
       else:
-        filters = [subsuite(options.subsuite)]
-        tests = manifest.active_tests(
-            disabled=disabled, filters=filters, **info)
+        tests = manifest.active_tests(disabled=disabled, options=options, **info)
         if len(tests) == 0:
-          tests = manifest.active_tests(disabled=True, **info)
+          tests = manifest.active_tests(disabled=True, options=options, **info)
 
     paths = []
 
     for test in tests:
       if len(tests) == 1 and 'disabled' in test:
         del test['disabled']
 
       pathAbs = os.path.abspath(test['path'])
--- a/testing/mozbase/docs/conf.py
+++ b/testing/mozbase/docs/conf.py
@@ -93,25 +93,16 @@ pygments_style = 'sphinx'
 #modindex_common_prefix = []
 
 
 # -- Options for HTML output ---------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 html_theme = 'default'
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-
-if not on_rtd:
-    try:
-        import sphinx_rtd_theme
-        html_theme = 'sphinx_rtd_theme'
-        html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-    except ImportError:
-        pass
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
 #html_theme_path = []
--- a/testing/mozbase/docs/manifestparser.rst
+++ b/testing/mozbase/docs/manifestparser.rst
@@ -1,13 +1,11 @@
 Managing lists of tests
 =======================
 
-.. py:currentmodule:: manifestparser
-
 We don't always want to run all tests, all the time. Sometimes a test
 may be broken, in other cases we only want to run a test on a specific
 platform or build of Mozilla. To handle these cases (and more), we
 created a python library to create and use test "manifests", which
 codify this information.
 
 :mod:`manifestparser` --- Create and manage test manifests
 -----------------------------------------------------------
@@ -259,83 +257,43 @@ 3. Optionally, a harness will have an in
 from TestManifest if more harness-specific customization is desired at
 the manifest level.
 
 See the source code at https://github.com/mozilla/mozbase/tree/master/manifestparser
 and
 https://github.com/mozilla/mozbase/blob/master/manifestparser/manifestparser.py
 in particular.
 
-Filtering Manifests
-```````````````````
-
-After creating a `TestManifest` object, all manifest files are read and a list
-of test objects can be accessed via `TestManifest.tests`. However this list contains
-all test objects, whether they should be run or not. Normally they need to be
-filtered down only to the set of tests that should be run by the test harness.
-
-To do this, a test harness can call `TestManifest.active_tests`:
-
-.. code-block:: python
-
-    tests = manifest.active_tests(exists=True, disabled=True, **tags)
-
-By default, `active_tests` runs the filters found in
-:attr:`~.DEFAULT_FILTERS`. It also accepts two convenience arguments:
-
-1. `exists`: if True (default), filter out tests that do not exist on the local file system.
-2. `disabled`: if True (default), do not filter out tests containing the 'disabled' key
-   (which can be set by `skip-if`, `run-if` or manually).
+Using Manifests
+```````````````
 
-This works for simple cases, but there are other built-in filters, or even custom filters
-that can be applied to the `TestManifest`. To do so, add the filter to `TestManifest.filters`:
-
-.. code-block:: python
-
-    from manifestparser.filters import subsuite
-    import mozinfo
-
-    filters = [subsuite('devtools')]
-    tests = manifest.active_tests(filters=filters, **mozinfo.info)
-
-.. automodule:: manifestparser.filters
-    :members:
-    :exclude-members: filterlist,InstanceFilter,DEFAULT_FILTERS
-
-.. autodata::  manifestparser.filters.DEFAULT_FILTERS
-    :annotation:
-
-For example, suppose we want to introduce a new key called `timeout-if` that adds a
-'timeout' property to a test if a certain condition is True. The syntax in the manifest
-files will look like this:
+A test harness will normally call `TestManifest.active_tests`:
 
 .. code-block:: text
 
-    [test_foo.py]
-    timeout-if = 300, os == 'win'
+    def active_tests(self, exists=True, disabled=True, **tags):
 
-The value is <timeout>, <condition> where condition is the same format as the one in
-`skip-if`. In the above case, if os == 'win', a timeout of 300 seconds will be
-applied. Otherwise, no timeout will be applied. All we need to do is define the filter
-and add it:
-
-.. code-block:: python
+The manifests are passed to the `__init__` or `read` methods with
+appropriate arguments.  `active_tests` then allows you to select the
+tests you want:
 
-    from manifestparser.expression import parse
-    import mozinfo
+- exists : return only existing tests
+- disabled : whether to return disabled tests; if not these will be
+  filtered out; if True (the default), the `disabled` key of a
+  test's metadata will be present and will be set to the reason that a
+  test is disabled
+- tags : keys and values to filter on (e.g. `os='linux'`)
 
-    def timeout_if(tests, values):
-        for test in tests:
-            if 'timeout-if' in test:
-                timeout, condition = test['timeout-if'].split(',', 1)
-                if parse(condition, **values):
-                    test['timeout'] = timeout
-            yield test
-
-    tests = manifest.active_tests(filters=[timeout_if], **mozinfo.info)
+`active_tests` looks for tests with `skip-if`
+`run-if`.  If the condition is or is not fulfilled,
+respectively, the test is marked as disabled.  For instance, if you
+pass `**dict(os='linux')` as `**tags`, if a test contains a line
+`skip-if = os == 'linux'` this test will be disabled, or
+`run-if = os = 'win'` in which case the test will also be disabled.  It
+is up to the harness to pass in tags appropriate to its usage.
 
 Creating Manifests
 ``````````````````
 
 manifestparser comes with a console script, `manifestparser create`, that
 may be used to create a seed manifest structure from a directory of
 files.  Run `manifestparser help create` for usage information.
 
deleted file mode 100644
--- a/testing/mozbase/manifestparser/manifestparser/filters.py
+++ /dev/null
@@ -1,179 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-"""
-A filter is a callable that accepts an iterable of test objects and a
-dictionary of values, and returns a new iterable of test objects. It is
-possible to define custom filters if the built-in ones are not enough.
-"""
-
-from collections import MutableSequence
-import os
-
-from .expression import (
-    parse,
-    ParseError,
-)
-
-
-# built-in filters
-
-def skip_if(tests, values):
-    """
-    Sets disabled on all tests containing the `skip-if` tag and whose condition
-    is True. This filter is added by default.
-    """
-    tag = 'skip-if'
-    for test in tests:
-        if tag in test and parse(test[tag], **values):
-            test.setdefault('disabled', '{}: {}'.format(tag, test[tag]))
-        yield test
-
-
-def run_if(tests, values):
-    """
-    Sets disabled on all tests containing the `run-if` tag and whose condition
-    is False. This filter is added by default.
-    """
-    tag = 'run-if'
-    for test in tests:
-        if tag in test and not parse(test[tag], **values):
-            test.setdefault('disabled', '{}: {}'.format(tag, test[tag]))
-        yield test
-
-
-def fail_if(tests, values):
-    """
-    Sets expected to 'fail' on all tests containing the `fail-if` tag and whose
-    condition is True. This filter is added by default.
-    """
-    tag = 'fail-if'
-    for test in tests:
-        if tag in test and parse(test[tag], **values):
-            test['expected'] = 'fail'
-        yield test
-
-
-def enabled(tests, values):
-    """
-    Removes all tests containing the `disabled` key. This filter can be
-    added by passing `disabled=False` into `active_tests`.
-    """
-    for test in tests:
-        if 'disabled' not in test:
-            yield test
-
-
-def exists(tests, values):
-    """
-    Removes all tests that do not exist on the file system. This filter is
-    added by default, but can be removed by passing `exists=False` into
-    `active_tests`.
-    """
-    for test in tests:
-        if os.path.exists(test['path']):
-            yield test
-
-
-# built-in instance filters
-
-class InstanceFilter(object):
-    """
-    Generally only one instance of a class filter should be applied at a time.
-    Two instances of `InstanceFilter` are considered equal if they have the
-    same class name. This ensures only a single instance is ever added to
-    `filterlist`.
-    """
-    def __eq__(self, other):
-        return self.__class__ == other.__class__
-
-
-class subsuite(InstanceFilter):
-    """
-    If `name` is None, removes all tests that have a `subsuite` key.
-    Otherwise removes all tests that do not have a subsuite matching `name`.
-
-    It is possible to specify conditional subsuite keys using:
-       subsuite = foo,condition
-
-    where 'foo' is the subsuite name, and 'condition' is the same type of
-    condition used for skip-if.  If the condition doesn't evaluate to true,
-    the subsuite designation will be removed from the test.
-
-    :param name: The name of the subsuite to run (default None)
-    """
-    def __init__(self, name=None):
-        self.name = name
-
-    def __call__(self, tests, values):
-        # Look for conditional subsuites, and replace them with the subsuite
-        # itself (if the condition is true), or nothing.
-        for test in tests:
-            subsuite = test.get('subsuite', '')
-            if ',' in subsuite:
-                try:
-                    subsuite, cond = subsuite.split(',')
-                except ValueError:
-                    raise ParseError("subsuite condition can't contain commas")
-                matched = parse(cond, **values)
-                if matched:
-                    test['subsuite'] = subsuite
-                else:
-                    test['subsuite'] = ''
-
-            # Filter on current subsuite
-            if self.name is None:
-                if not test.get('subsuite'):
-                    yield test
-            else:
-                if test.get('subsuite') == self.name:
-                    yield test
-
-
-# filter container
-
-DEFAULT_FILTERS = (
-    skip_if,
-    run_if,
-    fail_if,
-)
-"""
-By default :func:`~.active_tests` will run the :func:`~.skip_if`,
-:func:`~.run_if` and :func:`~.fail_if` filters.
-"""
-
-
-class filterlist(MutableSequence):
-    """
-    A MutableSequence that raises TypeError when adding a non-callable and
-    ValueError if the item is already added.
-    """
-
-    def __init__(self, items=None):
-        self.items = []
-        if items:
-            self.items = list(items)
-
-    def _validate(self, item):
-        if not callable(item):
-            raise TypeError("Filters must be callable!")
-        if item in self:
-            raise ValueError("Filter {} is already applied!".format(item))
-
-    def __getitem__(self, key):
-        return self.items[key]
-
-    def __setitem__(self, key, value):
-        self._validate(value)
-        self.items[key] = value
-
-    def __delitem__(self, key):
-        del self.items[key]
-
-    def __len__(self):
-        return len(self.items)
-
-    def insert(self, index, value):
-        self._validate(value)
-        self.items.insert(index, value)
old mode 100644
new mode 100755
--- a/testing/mozbase/manifestparser/manifestparser/manifestparser.py
+++ b/testing/mozbase/manifestparser/manifestparser/manifestparser.py
@@ -7,21 +7,19 @@
 from StringIO import StringIO
 import json
 import fnmatch
 import os
 import shutil
 import sys
 
 from .ini import read_ini
-from .filters import (
-    DEFAULT_FILTERS,
-    enabled,
-    exists as _exists,
-    filterlist,
+from .expression import (
+    parse,
+    ParseError,
 )
 
 relpath = os.path.relpath
 string = (basestring,)
 
 
 ### path normalization
 
@@ -308,23 +306,21 @@ class ManifestParser(object):
 
     def paths(self):
         return [i['path'] for i in self.tests]
 
 
     ### methods for auditing
 
     def missing(self, tests=None):
-        """
-        return list of tests that do not exist on the filesystem
-        """
+        """return list of tests that do not exist on the filesystem"""
         if tests is None:
             tests = self.tests
-        existing = _exists(tests, {})
-        return [t for t in tests if t not in existing]
+        return [test for test in tests
+                if not os.path.exists(test['path'])]
 
     def check_missing(self, tests=None):
         missing = self.missing(tests=tests)
         if missing:
             missing_paths = [test['path'] for test in missing]
             if self.strict:
                 raise IOError("Strict mode enabled, test paths must exist. "
                               "The following test(s) are missing: %s" %
@@ -702,48 +698,120 @@ convert = ManifestParser.from_directorie
 
 
 class TestManifest(ManifestParser):
     """
     apply logic to manifests;  this is your integration layer :)
     specific harnesses may subclass from this if they need more logic
     """
 
-    def __init__(self, *args, **kwargs):
-        ManifestParser.__init__(self, *args, **kwargs)
-        self.filters = filterlist(DEFAULT_FILTERS)
-
-    def active_tests(self, exists=True, disabled=True, filters=None, **values):
+    def filter(self, values, tests):
+        """
+        filter on a specific list tag, e.g.:
+        run-if = os == win linux
+        skip-if = os == mac
         """
-        Run all applied filters on the set of tests.
+
+        # tags:
+        run_tag = 'run-if'
+        skip_tag = 'skip-if'
+        fail_tag = 'fail-if'
+
+        cache = {}
+
+        def _parse(cond):
+            if '#' in cond:
+                cond = cond[:cond.index('#')]
+            cond = cond.strip()
+            if cond in cache:
+                ret = cache[cond]
+            else:
+                ret = parse(cond, **values)
+                cache[cond] = ret
+            return ret
+
+        # loop over test
+        for test in tests:
+            reason = None # reason to disable
 
-        :param exists: filter out non-existing tests (default True)
-        :param disabled: whether to return disabled tests (default True)
-        :param values: keys and values to filter on (e.g. `os = linux mac`)
-        :param filters: list of filters to apply to the tests
-        :returns: list of test objects that were not filtered out
+            # tagged-values to run
+            if run_tag in test:
+                condition = test[run_tag]
+                if not _parse(condition):
+                    reason = '%s: %s' % (run_tag, condition)
+
+            # tagged-values to skip
+            if skip_tag in test:
+                condition = test[skip_tag]
+                if _parse(condition):
+                    reason = '%s: %s' % (skip_tag, condition)
+
+            # mark test as disabled if there's a reason
+            if reason:
+                test.setdefault('disabled', reason)
+
+            # mark test as a fail if so indicated
+            if fail_tag in test:
+                condition = test[fail_tag]
+                if _parse(condition):
+                    test['expected'] = 'fail'
+
+    def active_tests(self, exists=True, disabled=True, options=None, **values):
+        """
+        - exists : return only existing tests
+        - disabled : whether to return disabled tests
+        - options: an optparse or argparse options object, used for subsuites
+        - values : keys and values to filter on (e.g. `os = linux mac`)
         """
         tests = [i.copy() for i in self.tests] # shallow copy
 
-        # mark all tests as passing
+        # Conditional subsuites are specified using:
+        #    subsuite = foo,condition
+        # where 'foo' is the subsuite name, and 'condition' is the same type of
+        # condition used for skip-if.  If the condition doesn't evaluate to true,
+        # the subsuite designation will be removed from the test.
+        #
+        # Look for conditional subsuites, and replace them with the subsuite itself
+        # (if the condition is true), or nothing.
+        for test in tests:
+            subsuite = test.get('subsuite', '')
+            if ',' in subsuite:
+                try:
+                    subsuite, condition = subsuite.split(',')
+                except ValueError:
+                    raise ParseError("subsuite condition can't contain commas")
+                # strip any comments from the condition
+                condition = condition.split('#')[0]
+                matched = parse(condition, **values)
+                if matched:
+                    test['subsuite'] = subsuite
+                else:
+                    test['subsuite'] = ''
+
+        # Filter on current subsuite
+        if options:
+            if hasattr(options, 'subsuite') and options.subsuite:
+                tests = [test for test in tests if options.subsuite == test['subsuite']]
+            else:
+                tests = [test for test in tests if not test['subsuite']]
+
+        # mark all tests as passing unless indicated otherwise
         for test in tests:
             test['expected'] = test.get('expected', 'pass')
 
-        # make a copy so original doesn't get modified
-        fltrs = self.filters[:]
+        # ignore tests that do not exist
         if exists:
-            if self.strict:
-                self.check_missing(tests)
-            else:
-                fltrs.append(_exists)
+            missing = self.check_missing(tests)
+            tests = [test for test in tests if test not in missing]
 
+        # filter by tags
+        self.filter(values, tests)
+
+        # ignore disabled tests if specified
         if not disabled:
-            fltrs.append(enabled)
+            tests = [test for test in tests
+                     if not 'disabled' in test]
 
-        if filters:
-            fltrs += filters
-
-        for fn in fltrs:
-            tests = fn(tests, values)
-        return list(tests)
+        # return active tests
+        return tests
 
     def test_paths(self):
         return [test['path'] for test in self.active_tests()]
--- a/testing/mozbase/manifestparser/setup.py
+++ b/testing/mozbase/manifestparser/setup.py
@@ -1,16 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from setuptools import setup
 
 PACKAGE_NAME = "manifestparser"
-PACKAGE_VERSION = '1.0'
+PACKAGE_VERSION = '0.9'
 
 setup(name=PACKAGE_NAME,
       version=PACKAGE_VERSION,
       description="Library to create and manage test manifests",
       long_description="see http://mozbase.readthedocs.org/",
       classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
       keywords='mozilla manifests',
       author='Mozilla Automation and Testing Team',
--- a/testing/mozbase/manifestparser/tests/manifest.ini
+++ b/testing/mozbase/manifestparser/tests/manifest.ini
@@ -1,11 +1,9 @@
 # test manifest for manifestparser
 [test_expressionparser.py]
 [test_manifestparser.py]
 [test_testmanifest.py]
 [test_read_ini.py]
 [test_convert_directory.py]
-[test_filters.py]
-[test_chunking.py]
 
 [test_convert_symlinks.py]
 disabled = https://bugzilla.mozilla.org/show_bug.cgi?id=920938
deleted file mode 100644
--- a/testing/mozbase/manifestparser/tests/test_filters.py
+++ /dev/null
@@ -1,156 +0,0 @@
-#!/usr/bin/env python
-
-from copy import deepcopy
-import os
-import unittest
-
-from manifestparser import TestManifest
-from manifestparser.filters import (
-    subsuite,
-    skip_if,
-    run_if,
-    fail_if,
-    enabled,
-    exists,
-    filterlist,
-)
-
-here = os.path.dirname(os.path.abspath(__file__))
-
-
-class FilterList(unittest.TestCase):
-    """Test filterlist datatype"""
-
-    def test_data_model(self):
-        foo = lambda x, y: x
-        bar = lambda x, y: x
-        baz = lambda x, y: x
-        fl = filterlist()
-
-        fl.extend([foo, bar])
-        self.assertEquals(len(fl), 2)
-        self.assertTrue(foo in fl)
-
-        fl.append(baz)
-        self.assertEquals(fl[2], baz)
-
-        fl.remove(baz)
-        self.assertFalse(baz in fl)
-
-        item = fl.pop()
-        self.assertEquals(item, bar)
-
-        self.assertEquals(fl.index(foo), 0)
-
-        del fl[0]
-        self.assertFalse(foo in fl)
-        with self.assertRaises(IndexError):
-            fl[0]
-
-    def test_add_non_callable_to_set(self):
-        fl = filterlist()
-        with self.assertRaises(TypeError):
-            fl.append('foo')
-
-    def test_add_duplicates_to_set(self):
-        foo = lambda x, y: x
-        bar = lambda x, y: x
-        sub = subsuite('foo')
-        fl = filterlist([foo, bar, sub])
-        self.assertEquals(len(fl), 3)
-        self.assertEquals(fl[0], foo)
-
-        with self.assertRaises(ValueError):
-            fl.append(foo)
-
-        with self.assertRaises(ValueError):
-            fl.append(subsuite('bar'))
-
-    def test_filters_run_in_order(self):
-        a = lambda x, y: x
-        b = lambda x, y: x
-        c = lambda x, y: x
-        d = lambda x, y: x
-        e = lambda x, y: x
-        f = lambda x, y: x
-
-        fl = filterlist([a, b])
-        fl.append(c)
-        fl.extend([d, e])
-        fl += [f]
-        self.assertEquals([i for i in fl], [a, b, c, d, e, f])
-
-
-class BuiltinFilters(unittest.TestCase):
-    """Test the built-in filters"""
-
-    tests = (
-        { "name": "test0" },
-        { "name": "test1", "skip-if": "foo == 'bar'" },
-        { "name": "test2", "run-if": "foo == 'bar'" },
-        { "name": "test3", "fail-if": "foo == 'bar'" },
-        { "name": "test4", "disabled": "some reason" },
-        { "name": "test5", "subsuite": "baz" },
-        { "name": "test6", "subsuite": "baz,foo == 'bar'" })
-
-    def test_skip_if(self):
-        tests = deepcopy(self.tests)
-        tests = list(skip_if(tests, {}))
-        self.assertEquals(len(tests), len(self.tests))
-
-        tests = deepcopy(self.tests)
-        tests = list(skip_if(tests, {'foo': 'bar'}))
-        self.assertNotIn(self.tests[1], tests)
-
-    def test_run_if(self):
-        tests = deepcopy(self.tests)
-        tests = list(run_if(tests, {}))
-        self.assertNotIn(self.tests[2], tests)
-
-        tests = deepcopy(self.tests)
-        tests = list(run_if(tests, {'foo': 'bar'}))
-        self.assertEquals(len(tests), len(self.tests))
-
-    def test_fail_if(self):
-        tests = deepcopy(self.tests)
-        tests = list(fail_if(tests, {}))
-        self.assertNotIn('expected', tests[3])
-
-        tests = deepcopy(self.tests)
-        tests = list(fail_if(tests, {'foo': 'bar'}))
-        self.assertEquals(tests[3]['expected'], 'fail')
-
-    def test_enabled(self):
-        tests = deepcopy(self.tests)
-        tests = list(enabled(tests, {}))
-        self.assertNotIn(self.tests[4], tests)
-
-    def test_subsuite(self):
-        sub1 = subsuite()
-        sub2 = subsuite('baz')
-
-        tests = deepcopy(self.tests)
-        tests = list(sub1(tests, {}))
-        self.assertNotIn(self.tests[5], tests)
-        self.assertEquals(tests[-1]['name'], 'test6')
-
-        tests = deepcopy(self.tests)
-        tests = list(sub2(tests, {}))
-        self.assertEquals(len(tests), 1)
-        self.assertIn(self.tests[5], tests)
-
-    def test_subsuite_condition(self):
-        sub1 = subsuite()
-        sub2 = subsuite('baz')
-
-        tests = deepcopy(self.tests)
-
-        tests = list(sub1(tests, {'foo': 'bar'}))
-        self.assertNotIn(self.tests[5], tests)
-        self.assertNotIn(self.tests[6], tests)
-
-        tests = deepcopy(self.tests)
-        tests = list(sub2(tests, {'foo': 'bar'}))
-        self.assertEquals(len(tests), 2)
-        self.assertEquals(tests[0]['name'], 'test5')
-        self.assertEquals(tests[1]['name'], 'test6')
--- a/testing/mozbase/manifestparser/tests/test_testmanifest.py
+++ b/testing/mozbase/manifestparser/tests/test_testmanifest.py
@@ -1,17 +1,15 @@
 #!/usr/bin/env python
 
 import os
 import shutil
 import tempfile
 import unittest
-
 from manifestparser import TestManifest, ParseError
-from manifestparser.filters import subsuite
 
 here = os.path.dirname(os.path.abspath(__file__))
 
 
 class TestTestManifest(unittest.TestCase):
     """Test the Test Manifest"""
 
     def test_testmanifest(self):
@@ -23,21 +21,21 @@ class TestTestManifest(unittest.TestCase
         self.assertEqual([i['name'] for i in manifest.active_tests(os='linux', disabled=False, exists=False)],
                          ['fleem', 'linuxtest'])
 
         # Look for existing tests.  There is only one:
         self.assertEqual([i['name'] for i in manifest.active_tests()],
                          ['fleem'])
 
         # You should be able to expect failures:
-        last = manifest.active_tests(exists=False, toolkit='gtk2')[-1]
-        self.assertEqual(last['name'], 'linuxtest')
-        self.assertEqual(last['expected'], 'pass')
-        last = manifest.active_tests(exists=False, toolkit='cocoa')[-1]
-        self.assertEqual(last['expected'], 'fail')
+        last_test = manifest.active_tests(exists=False, toolkit='gtk2')[-1]
+        self.assertEqual(last_test['name'], 'linuxtest')
+        self.assertEqual(last_test['expected'], 'pass')
+        last_test = manifest.active_tests(exists=False, toolkit='cocoa')[-1]
+        self.assertEqual(last_test['expected'], 'fail')
 
     def test_missing_paths(self):
         """
         Test paths that don't exist raise an exception in strict mode.
         """
         tempdir = tempfile.mkdtemp()
 
         missing_path = os.path.join(here, 'missing-path.ini')
@@ -58,55 +56,57 @@ class TestTestManifest(unittest.TestCase
         self.assertEqual(len(manifest.tests), 8)
         names = [i['name'] for i in manifest.tests]
         self.assertFalse('test_0202_app_launch_apply_update_dirlocked.js' in names)
 
     def test_manifest_subsuites(self):
         """
         test subsuites and conditional subsuites
         """
+        class AttributeDict(dict):
+            def __getattr__(self, attr):
+                return self[attr]
+            def __setattr__(self, attr, value):
+                self[attr] = value
+
         relative_path = os.path.join(here, 'subsuite.ini')
         manifest = TestManifest(manifests=(relative_path,))
         info = {'foo': 'bar'}
+        options = {'subsuite': 'bar'}
 
         # 6 tests total
-        tests = manifest.active_tests(exists=False, **info)
-        self.assertEquals(len(tests), 6)
+        self.assertEquals(len(manifest.active_tests(exists=False, **info)), 6)
 
         # only 3 tests for subsuite bar when foo==bar
-        tests = manifest.active_tests(exists=False,
-                                      filters=[subsuite('bar')],
-                                      **info)
-        self.assertEquals(len(tests), 3)
+        self.assertEquals(len(manifest.active_tests(exists=False,
+                                                    options=AttributeDict(options),
+                                                    **info)), 3)
 
-        # only 1 test for subsuite baz, regardless of conditions
+        options = {'subsuite': 'baz'}
         other = {'something': 'else'}
-        tests = manifest.active_tests(exists=False,
-                                      filters=[subsuite('baz')],
-                                      **info)
-        self.assertEquals(len(tests), 1)
-        tests = manifest.active_tests(exists=False,
-                                      filters=[subsuite('baz')],
-                                      **other)
-        self.assertEquals(len(tests), 1)
+        # only 1 test for subsuite baz, regardless of conditions
+        self.assertEquals(len(manifest.active_tests(exists=False,
+                                                    options=AttributeDict(options),
+                                                    **info)), 1)
+        self.assertEquals(len(manifest.active_tests(exists=False,
+                                                    options=AttributeDict(options),
+                                                    **other)), 1)
 
         # 4 tests match when the condition doesn't match (all tests except
         # the unconditional subsuite)
         info = {'foo': 'blah'}
-        tests = manifest.active_tests(exists=False,
-                                      filters=[subsuite()],
-                                      **info)
-        self.assertEquals(len(tests), 5)
+        options = {'subsuite': None}
+        self.assertEquals(len(manifest.active_tests(exists=False,
+                                                    options=AttributeDict(options),
+                                                    **info)), 5)
 
         # test for illegal subsuite value
         manifest.tests[0]['subsuite'] = 'subsuite=bar,foo=="bar",type="nothing"'
-        with self.assertRaises(ParseError):
-            manifest.active_tests(exists=False,
-                                  filters=[subsuite('foo')],
-                                  **info)
+        self.assertRaises(ParseError, manifest.active_tests, exists=False,
+                          options=AttributeDict(options), **info)
 
     def test_none_and_empty_manifest(self):
         """
         Test TestManifest for None and empty manifest, see
         https://bugzilla.mozilla.org/show_bug.cgi?id=1087682
         """
         none_manifest = TestManifest(manifests=None, strict=False)
         self.assertEqual(len(none_manifest.test_paths()), 0)