Bug 1120983 - [manifestparser] Split manifestparser.py into several smaller files, r=wlach
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 15 Jan 2015 09:37:51 -0500
changeset 238457 efc0e28680fd09ae7d969d99fd68f8b34b96d9bd
parent 238456 51f6b5ca2e47396f0b58372e48c9f1f8613c0ef8
child 238458 b00198e105b62eea74c930a817c9d10b9eea98bd
push id4610
push userjlund@mozilla.com
push dateMon, 30 Mar 2015 18:32:55 +0000
treeherdermozilla-esr52@4df54044d9ef [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerswlach
bugs1120983
milestone38.0a1
Bug 1120983 - [manifestparser] Split manifestparser.py into several smaller files, r=wlach Simple refactor that moves logic out of manifestparser.py and into cli.py, expression.py and ini.py.
testing/mochitest/moz.build
testing/mozbase/manifestparser/manifestparser/__init__.py
testing/mozbase/manifestparser/manifestparser/cli.py
testing/mozbase/manifestparser/manifestparser/expression.py
testing/mozbase/manifestparser/manifestparser/ini.py
testing/mozbase/manifestparser/manifestparser/manifestparser.py
testing/mozbase/manifestparser/setup.py
--- a/testing/mochitest/moz.build
+++ b/testing/mochitest/moz.build
@@ -28,17 +28,16 @@ MOCHITEST_CHROME_MANIFESTS += ['chrome/c
 
 TEST_HARNESS_FILES.testing.mochitest += [
     '!automation.py',
     '/build/automationutils.py',
     '/build/mobile/remoteautomation.py',
     '/build/pgo/server-locations.txt',
     '/build/sanitizers/lsan_suppressions.txt',
     '/netwerk/test/httpserver/httpd.js',
-    '/testing/mozbase/manifestparser/manifestparser/manifestparser.py',
     '/testing/mozbase/mozdevice/mozdevice/devicemanager.py',
     '/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py',
     '/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py',
     '/testing/mozbase/mozdevice/mozdevice/droid.py',
     '/testing/mozbase/mozdevice/mozdevice/Zeroconf.py',
     '/testing/mozbase/moznetwork/moznetwork/moznetwork.py',
     'android.json',
     'android23.json',
--- a/testing/mozbase/manifestparser/manifestparser/__init__.py
+++ b/testing/mozbase/manifestparser/manifestparser/__init__.py
@@ -1,5 +1,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from manifestparser import *
+from .manifestparser import *
+from .expression import *
+from .ini import *
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/manifestparser/manifestparser/cli.py
@@ -0,0 +1,237 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Mozilla universal manifest parser
+"""
+
+from optparse import OptionParser
+import os
+import sys
+
+from .manifestparser import (
+    convert,
+    ManifestParser,
+)
+
+
+class ParserError(Exception):
+  """error for exceptions while parsing the command line"""
+
+def parse_args(_args):
+    """
+    parse and return:
+    --keys=value (or --key value)
+    -tags
+    args
+    """
+
+    # return values
+    _dict = {}
+    tags = []
+    args = []
+
+    # parse the arguments
+    key = None
+    for arg in _args:
+        if arg.startswith('---'):
+            raise ParserError("arguments should start with '-' or '--' only")
+        elif arg.startswith('--'):
+            if key:
+                raise ParserError("Key %s still open" % key)
+            key = arg[2:]
+            if '=' in key:
+                key, value = key.split('=', 1)
+                _dict[key] = value
+                key = None
+                continue
+        elif arg.startswith('-'):
+            if key:
+                raise ParserError("Key %s still open" % key)
+            tags.append(arg[1:])
+            continue
+        else:
+            if key:
+                _dict[key] = arg
+                continue
+            args.append(arg)
+
+    # return values
+    return (_dict, tags, args)
+
+class CLICommand(object):
+    usage = '%prog [options] command'
+    def __init__(self, parser):
+      self._parser = parser # master parser
+    def parser(self):
+      return OptionParser(usage=self.usage, description=self.__doc__,
+                          add_help_option=False)
+
+class Copy(CLICommand):
+    usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
+    def __call__(self, options, args):
+      # parse the arguments
+      try:
+        kwargs, tags, args = parse_args(args)
+      except ParserError, e:
+        self._parser.error(e.message)
+
+      # make sure we have some manifests, otherwise it will
+      # be quite boring
+      if not len(args) == 2:
+        HelpCLI(self._parser)(options, ['copy'])
+        return
+
+      # read the manifests
+      # TODO: should probably ensure these exist here
+      manifests = ManifestParser()
+      manifests.read(args[0])
+
+      # print the resultant query
+      manifests.copy(args[1], None, *tags, **kwargs)
+
+
+class CreateCLI(CLICommand):
+    """
+    create a manifest from a list of directories
+    """
+    usage = '%prog [options] create directory <directory> <...>'
+
+    def parser(self):
+        parser = CLICommand.parser(self)
+        parser.add_option('-p', '--pattern', dest='pattern',
+                          help="glob pattern for files")
+        parser.add_option('-i', '--ignore', dest='ignore',
+                          default=[], action='append',
+                          help='directories to ignore')
+        parser.add_option('-w', '--in-place', dest='in_place',
+                          help='Write .ini files in place; filename to write to')
+        return parser
+
+    def __call__(self, _options, args):
+        parser = self.parser()
+        options, args = parser.parse_args(args)
+
+        # need some directories
+        if not len(args):
+            parser.print_usage()
+            return
+
+        # add the directories to the manifest
+        for arg in args:
+            assert os.path.exists(arg)
+            assert os.path.isdir(arg)
+            manifest = convert(args, pattern=options.pattern, ignore=options.ignore,
+                               write=options.in_place)
+        if manifest:
+            print manifest
+
+
+class WriteCLI(CLICommand):
+    """
+    write a manifest based on a query
+    """
+    usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...'
+    def __call__(self, options, args):
+
+        # parse the arguments
+        try:
+            kwargs, tags, args = parse_args(args)
+        except ParserError, e:
+            self._parser.error(e.message)
+
+        # make sure we have some manifests, otherwise it will
+        # be quite boring
+        if not args:
+            HelpCLI(self._parser)(options, ['write'])
+            return
+
+        # read the manifests
+        # TODO: should probably ensure these exist here
+        manifests = ManifestParser()
+        manifests.read(*args)
+
+        # print the resultant query
+        manifests.write(global_tags=tags, global_kwargs=kwargs)
+
+
+
+class HelpCLI(CLICommand):
+    """
+    get help on a command
+    """
+    usage = '%prog [options] help [command]'
+
+    def __call__(self, options, args):
+        if len(args) == 1 and args[0] in commands:
+            commands[args[0]](self._parser).parser().print_help()
+        else:
+            self._parser.print_help()
+            print '\nCommands:'
+            for command in sorted(commands):
+                print '  %s : %s' % (command, commands[command].__doc__.strip())
+
+class UpdateCLI(CLICommand):
+    """
+    update the tests as listed in a manifest from a directory
+    """
+    usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
+
+    def __call__(self, options, args):
+        # parse the arguments
+        try:
+            kwargs, tags, args = parse_args(args)
+        except ParserError, e:
+            self._parser.error(e.message)
+
+        # make sure we have some manifests, otherwise it will
+        # be quite boring
+        if not len(args) == 2:
+            HelpCLI(self._parser)(options, ['update'])
+            return
+
+        # read the manifests
+        # TODO: should probably ensure these exist here
+        manifests = ManifestParser()
+        manifests.read(args[0])
+
+        # print the resultant query
+        manifests.update(args[1], None, *tags, **kwargs)
+
+
+# command -> class mapping
+commands = { 'create': CreateCLI,
+             'help': HelpCLI,
+             'update': UpdateCLI,
+             'write': WriteCLI }
+
+def main(args=sys.argv[1:]):
+    """console_script entry point"""
+
+    # set up an option parser
+    usage = '%prog [options] [command] ...'
+    description = "%s. Use `help` to display commands" % __doc__.strip()
+    parser = OptionParser(usage=usage, description=description)
+    parser.add_option('-s', '--strict', dest='strict',
+                      action='store_true', default=False,
+                      help='adhere strictly to errors')
+    parser.disable_interspersed_args()
+
+    options, args = parser.parse_args(args)
+
+    if not args:
+        HelpCLI(parser)(options, args)
+        parser.exit()
+
+    # get the command
+    command = args[0]
+    if command not in commands:
+        parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command))
+
+    handler = commands[command](parser)
+    handler(options, args[1:])
+
+if __name__ == '__main__':
+    main()
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/manifestparser/manifestparser/expression.py
@@ -0,0 +1,252 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+
+__all__ = ['parse', 'ParseError', 'ExpressionParser']
+
+# expr.py
+# from:
+# http://k0s.org/mozilla/hg/expressionparser
+# http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser
+
+# Implements a top-down parser/evaluator for simple boolean expressions.
+# ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm
+#
+# Rough grammar:
+# expr := literal
+#       | '(' expr ')'
+#       | expr '&&' expr
+#       | expr '||' expr
+#       | expr '==' expr
+#       | expr '!=' expr
+# literal := BOOL
+#          | INT
+#          | STRING
+#          | IDENT
+# BOOL   := true|false
+# INT    := [0-9]+
+# STRING := "[^"]*"
+# IDENT  := [A-Za-z_]\w*
+
+# Identifiers take their values from a mapping dictionary passed as the second
+# argument.
+
+# Glossary (see above URL for details):
+# - nud: null denotation
+# - led: left detonation
+# - lbp: left binding power
+# - rbp: right binding power
+
+class ident_token(object):
+    def __init__(self, scanner, value):
+        self.value = value
+    def nud(self, parser):
+        # identifiers take their value from the value mappings passed
+        # to the parser
+        return parser.value(self.value)
+
+class literal_token(object):
+    def __init__(self, scanner, value):
+        self.value = value
+    def nud(self, parser):
+        return self.value
+
+class eq_op_token(object):
+    "=="
+    def led(self, parser, left):
+        return left == parser.expression(self.lbp)
+
+class neq_op_token(object):
+    "!="
+    def led(self, parser, left):
+        return left != parser.expression(self.lbp)
+
+class not_op_token(object):
+    "!"
+    def nud(self, parser):
+        return not parser.expression(100)
+
+class and_op_token(object):
+    "&&"
+    def led(self, parser, left):
+        right = parser.expression(self.lbp)
+        return left and right
+
+class or_op_token(object):
+    "||"
+    def led(self, parser, left):
+        right = parser.expression(self.lbp)
+        return left or right
+
+class lparen_token(object):
+    "("
+    def nud(self, parser):
+        expr = parser.expression()
+        parser.advance(rparen_token)
+        return expr
+
+class rparen_token(object):
+    ")"
+
+class end_token(object):
+    """always ends parsing"""
+
+### derived literal tokens
+
+class bool_token(literal_token):
+    def __init__(self, scanner, value):
+        value = {'true':True, 'false':False}[value]
+        literal_token.__init__(self, scanner, value)
+
+class int_token(literal_token):
+    def __init__(self, scanner, value):
+        literal_token.__init__(self, scanner, int(value))
+
+class string_token(literal_token):
+    def __init__(self, scanner, value):
+        literal_token.__init__(self, scanner, value[1:-1])
+
+precedence = [(end_token, rparen_token),
+              (or_op_token,),
+              (and_op_token,),
+              (eq_op_token, neq_op_token),
+              (lparen_token,),
+              ]
+for index, rank in enumerate(precedence):
+    for token in rank:
+        token.lbp = index # lbp = lowest left binding power
+
+class ParseError(Exception):
+    """error parsing conditional expression"""
+
+class ExpressionParser(object):
+    """
+    A parser for a simple expression language.
+
+    The expression language can be described as follows::
+
+        EXPRESSION ::= LITERAL | '(' EXPRESSION ')' | '!' EXPRESSION | EXPRESSION OP EXPRESSION
+        OP ::= '==' | '!=' | '&&' | '||'
+        LITERAL ::= BOOL | INT | IDENT | STRING
+        BOOL ::= 'true' | 'false'
+        INT ::= [0-9]+
+        IDENT ::= [a-zA-Z_]\w*
+        STRING ::= '"' [^\"] '"' | ''' [^\'] '''
+
+    At its core, expressions consist of booleans, integers, identifiers and.
+    strings. Booleans are one of *true* or *false*. Integers are a series
+    of digits. Identifiers are a series of English letters and underscores.
+    Strings are a pair of matching quote characters (single or double) with
+    zero or more characters inside.
+
+    Expressions can be combined with operators: the equals (==) and not
+    equals (!=) operators compare two expressions and produce a boolean. The
+    and (&&) and or (||) operators take two expressions and produce the logical
+    AND or OR value of them, respectively. An expression can also be prefixed
+    with the not (!) operator, which produces its logical negation.
+
+    Finally, any expression may be contained within parentheses for grouping.
+
+    Identifiers take their values from the mapping provided.
+    """
+
+    scanner = None
+
+    def __init__(self, text, valuemapping, strict=False):
+        """
+        Initialize the parser
+        :param text: The expression to parse as a string.
+        :param valuemapping: A dict mapping identifier names to values.
+        :param strict: If true, referencing an identifier that was not
+                       provided in :valuemapping: will raise an error.
+        """
+        self.text = text
+        self.valuemapping = valuemapping
+        self.strict = strict
+
+    def _tokenize(self):
+        """
+        Lex the input text into tokens and yield them in sequence.
+        """
+        if not ExpressionParser.scanner:
+            ExpressionParser.scanner = re.Scanner([
+                # Note: keep these in sync with the class docstring above.
+                (r"true|false", bool_token),
+                (r"[a-zA-Z_]\w*", ident_token),
+                (r"[0-9]+", int_token),
+                (r'("[^"]*")|(\'[^\']*\')', string_token),
+                (r"==", eq_op_token()),
+                (r"!=", neq_op_token()),
+                (r"\|\|", or_op_token()),
+                (r"!", not_op_token()),
+                (r"&&", and_op_token()),
+                (r"\(", lparen_token()),
+                (r"\)", rparen_token()),
+                (r"\s+", None), # skip whitespace
+            ])
+        tokens, remainder = ExpressionParser.scanner.scan(self.text)
+        for t in tokens:
+            yield t
+        yield end_token()
+
+    def value(self, ident):
+        """
+        Look up the value of |ident| in the value mapping passed in the
+        constructor.
+        """
+        if self.strict:
+            return self.valuemapping[ident]
+        else:
+            return self.valuemapping.get(ident, None)
+
+    def advance(self, expected):
+        """
+        Assert that the next token is an instance of |expected|, and advance
+        to the next token.
+        """
+        if not isinstance(self.token, expected):
+            raise Exception, "Unexpected token!"
+        self.token = self.iter.next()
+
+    def expression(self, rbp=0):
+        """
+        Parse and return the value of an expression until a token with
+        right binding power greater than rbp is encountered.
+        """
+        t = self.token
+        self.token = self.iter.next()
+        left = t.nud(self)
+        while rbp < self.token.lbp:
+            t = self.token
+            self.token = self.iter.next()
+            left = t.led(self, left)
+        return left
+
+    def parse(self):
+        """
+        Parse and return the value of the expression in the text
+        passed to the constructor. Raises a ParseError if the expression
+        could not be parsed.
+        """
+        try:
+            self.iter = self._tokenize()
+            self.token = self.iter.next()
+            return self.expression()
+        except:
+            raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping))
+
+    __call__ = parse
+
+
+def parse(text, **values):
+    """
+    Parse and evaluate a boolean expression.
+    :param text: The expression to parse, as a string.
+    :param values: A dict containing a name to value mapping for identifiers
+                   referenced in *text*.
+    :rtype: the final value of the expression.
+    :raises: :py:exc::ParseError: will be raised if parsing fails.
+    """
+    return ExpressionParser(text, values).parse()
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/manifestparser/manifestparser/ini.py
@@ -0,0 +1,125 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+__all__ = ['read_ini']
+
+import os
+
+def read_ini(fp, variables=None, default='DEFAULT', defaults_only=False,
+             comments=';#', separators=('=', ':'),
+             strict=True):
+    """
+    read an .ini file and return a list of [(section, values)]
+    - fp : file pointer or path to read
+    - variables : default set of variables
+    - default : name of the section for the default section
+    - defaults_only : if True, return the default section only
+    - comments : characters that if they start a line denote a comment
+    - separators : strings that denote key, value separation in order
+    - strict : whether to be strict about parsing
+    """
+
+    # variables
+    variables = variables or {}
+    sections = []
+    key = value = None
+    section_names = set()
+    if isinstance(fp, basestring):
+        fp = file(fp)
+
+    # read the lines
+    for (linenum, line) in enumerate(fp.readlines(), start=1):
+
+        stripped = line.strip()
+
+        # ignore blank lines
+        if not stripped:
+            # reset key and value to avoid continuation lines
+            key = value = None
+            continue
+
+        # ignore comment lines
+        if stripped[0] in comments:
+            continue
+
+        # check for a new section
+        if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']':
+            section = stripped[1:-1].strip()
+            key = value = None
+
+            # deal with DEFAULT section
+            if section.lower() == default.lower():
+                if strict:
+                    assert default not in section_names
+                section_names.add(default)
+                current_section = variables
+                continue
+
+            if strict:
+                # make sure this section doesn't already exist
+                assert section not in section_names, "Section '%s' already found in '%s'" % (section, section_names)
+
+            section_names.add(section)
+            current_section = {}
+            sections.append((section, current_section))
+            continue
+
+        # if there aren't any sections yet, something bad happen
+        if not section_names:
+            raise Exception('No sections found')
+
+        # (key, value) pair
+        for separator in separators:
+            if separator in stripped:
+                key, value = stripped.split(separator, 1)
+                key = key.strip()
+                value = value.strip()
+
+                if strict:
+                    # make sure this key isn't already in the section or empty
+                    assert key
+                    if current_section is not variables:
+                        assert key not in current_section
+
+                current_section[key] = value
+                break
+        else:
+            # continuation line ?
+            if line[0].isspace() and key:
+                value = '%s%s%s' % (value, os.linesep, stripped)
+                current_section[key] = value
+            else:
+                # something bad happened!
+                if hasattr(fp, 'name'):
+                    filename = fp.name
+                else:
+                    filename = 'unknown'
+                raise Exception("Error parsing manifest file '%s', line %s" %
+                                (filename, linenum))
+
+    # server-root is a special os path declared relative to the manifest file.
+    # inheritance demands we expand it as absolute
+    if 'server-root' in variables:
+        root = os.path.join(os.path.dirname(fp.name),
+                            variables['server-root'])
+        variables['server-root'] = os.path.abspath(root)
+
+    # return the default section only if requested
+    if defaults_only:
+        return [(default, variables)]
+
+    # interpret the variables
+    def interpret_variables(global_dict, local_dict):
+        variables = global_dict.copy()
+        if 'skip-if' in local_dict and 'skip-if' in variables:
+            local_dict['skip-if'] = "(%s) || (%s)" % (variables['skip-if'].split('#')[0], local_dict['skip-if'].split('#')[0])
+        variables.update(local_dict)
+
+        return variables
+
+    sections = [(i, interpret_variables(variables, j)) for i, j in sections]
+    return sections
+
+
+
--- a/testing/mozbase/manifestparser/manifestparser/manifestparser.py
+++ b/testing/mozbase/manifestparser/manifestparser/manifestparser.py
@@ -1,414 +1,46 @@
-#!/usr/bin/env python
-
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-"""
-Mozilla universal manifest parser
-"""
+__all__ = ['ManifestParser', 'TestManifest', 'convert']
 
-__all__ = ['read_ini', # .ini reader
-           'ManifestParser', 'TestManifest', 'convert', # manifest handling
-           'parse', 'ParseError', 'ExpressionParser'] # conditional expression parser
-
+from StringIO import StringIO
 import json
 import fnmatch
 import os
-import re
 import shutil
 import sys
 
-from optparse import OptionParser
-from StringIO import StringIO
+from .ini import read_ini
+from .expression import (
+    parse,
+    ParseError,
+)
 
 relpath = os.path.relpath
 string = (basestring,)
 
 
-# expr.py
-# from:
-# http://k0s.org/mozilla/hg/expressionparser
-# http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser
-
-# Implements a top-down parser/evaluator for simple boolean expressions.
-# ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm
-#
-# Rough grammar:
-# expr := literal
-#       | '(' expr ')'
-#       | expr '&&' expr
-#       | expr '||' expr
-#       | expr '==' expr
-#       | expr '!=' expr
-# literal := BOOL
-#          | INT
-#          | STRING
-#          | IDENT
-# BOOL   := true|false
-# INT    := [0-9]+
-# STRING := "[^"]*"
-# IDENT  := [A-Za-z_]\w*
-
-# Identifiers take their values from a mapping dictionary passed as the second
-# argument.
-
-# Glossary (see above URL for details):
-# - nud: null denotation
-# - led: left detonation
-# - lbp: left binding power
-# - rbp: right binding power
-
-class ident_token(object):
-    def __init__(self, scanner, value):
-        self.value = value
-    def nud(self, parser):
-        # identifiers take their value from the value mappings passed
-        # to the parser
-        return parser.value(self.value)
-
-class literal_token(object):
-    def __init__(self, scanner, value):
-        self.value = value
-    def nud(self, parser):
-        return self.value
-
-class eq_op_token(object):
-    "=="
-    def led(self, parser, left):
-        return left == parser.expression(self.lbp)
-
-class neq_op_token(object):
-    "!="
-    def led(self, parser, left):
-        return left != parser.expression(self.lbp)
-
-class not_op_token(object):
-    "!"
-    def nud(self, parser):
-        return not parser.expression(100)
-
-class and_op_token(object):
-    "&&"
-    def led(self, parser, left):
-        right = parser.expression(self.lbp)
-        return left and right
-
-class or_op_token(object):
-    "||"
-    def led(self, parser, left):
-        right = parser.expression(self.lbp)
-        return left or right
-
-class lparen_token(object):
-    "("
-    def nud(self, parser):
-        expr = parser.expression()
-        parser.advance(rparen_token)
-        return expr
-
-class rparen_token(object):
-    ")"
-
-class end_token(object):
-    """always ends parsing"""
-
-### derived literal tokens
-
-class bool_token(literal_token):
-    def __init__(self, scanner, value):
-        value = {'true':True, 'false':False}[value]
-        literal_token.__init__(self, scanner, value)
-
-class int_token(literal_token):
-    def __init__(self, scanner, value):
-        literal_token.__init__(self, scanner, int(value))
-
-class string_token(literal_token):
-    def __init__(self, scanner, value):
-        literal_token.__init__(self, scanner, value[1:-1])
-
-precedence = [(end_token, rparen_token),
-              (or_op_token,),
-              (and_op_token,),
-              (eq_op_token, neq_op_token),
-              (lparen_token,),
-              ]
-for index, rank in enumerate(precedence):
-    for token in rank:
-        token.lbp = index # lbp = lowest left binding power
-
-class ParseError(Exception):
-    """error parsing conditional expression"""
-
-class ExpressionParser(object):
-    """
-    A parser for a simple expression language.
-
-    The expression language can be described as follows::
-
-        EXPRESSION ::= LITERAL | '(' EXPRESSION ')' | '!' EXPRESSION | EXPRESSION OP EXPRESSION
-        OP ::= '==' | '!=' | '&&' | '||'
-        LITERAL ::= BOOL | INT | IDENT | STRING
-        BOOL ::= 'true' | 'false'
-        INT ::= [0-9]+
-        IDENT ::= [a-zA-Z_]\w*
-        STRING ::= '"' [^\"] '"' | ''' [^\'] '''
-
-    At its core, expressions consist of booleans, integers, identifiers and.
-    strings. Booleans are one of *true* or *false*. Integers are a series
-    of digits. Identifiers are a series of English letters and underscores.
-    Strings are a pair of matching quote characters (single or double) with
-    zero or more characters inside.
-
-    Expressions can be combined with operators: the equals (==) and not
-    equals (!=) operators compare two expressions and produce a boolean. The
-    and (&&) and or (||) operators take two expressions and produce the logical
-    AND or OR value of them, respectively. An expression can also be prefixed
-    with the not (!) operator, which produces its logical negation.
-
-    Finally, any expression may be contained within parentheses for grouping.
-
-    Identifiers take their values from the mapping provided.
-    """
-
-    scanner = None
-
-    def __init__(self, text, valuemapping, strict=False):
-        """
-        Initialize the parser
-        :param text: The expression to parse as a string.
-        :param valuemapping: A dict mapping identifier names to values.
-        :param strict: If true, referencing an identifier that was not
-                       provided in :valuemapping: will raise an error.
-        """
-        self.text = text
-        self.valuemapping = valuemapping
-        self.strict = strict
-
-    def _tokenize(self):
-        """
-        Lex the input text into tokens and yield them in sequence.
-        """
-        if not ExpressionParser.scanner:
-            ExpressionParser.scanner = re.Scanner([
-                # Note: keep these in sync with the class docstring above.
-                (r"true|false", bool_token),
-                (r"[a-zA-Z_]\w*", ident_token),
-                (r"[0-9]+", int_token),
-                (r'("[^"]*")|(\'[^\']*\')', string_token),
-                (r"==", eq_op_token()),
-                (r"!=", neq_op_token()),
-                (r"\|\|", or_op_token()),
-                (r"!", not_op_token()),
-                (r"&&", and_op_token()),
-                (r"\(", lparen_token()),
-                (r"\)", rparen_token()),
-                (r"\s+", None), # skip whitespace
-            ])
-        tokens, remainder = ExpressionParser.scanner.scan(self.text)
-        for t in tokens:
-            yield t
-        yield end_token()
-
-    def value(self, ident):
-        """
-        Look up the value of |ident| in the value mapping passed in the
-        constructor.
-        """
-        if self.strict:
-            return self.valuemapping[ident]
-        else:
-            return self.valuemapping.get(ident, None)
-
-    def advance(self, expected):
-        """
-        Assert that the next token is an instance of |expected|, and advance
-        to the next token.
-        """
-        if not isinstance(self.token, expected):
-            raise Exception, "Unexpected token!"
-        self.token = self.iter.next()
-
-    def expression(self, rbp=0):
-        """
-        Parse and return the value of an expression until a token with
-        right binding power greater than rbp is encountered.
-        """
-        t = self.token
-        self.token = self.iter.next()
-        left = t.nud(self)
-        while rbp < self.token.lbp:
-            t = self.token
-            self.token = self.iter.next()
-            left = t.led(self, left)
-        return left
-
-    def parse(self):
-        """
-        Parse and return the value of the expression in the text
-        passed to the constructor. Raises a ParseError if the expression
-        could not be parsed.
-        """
-        try:
-            self.iter = self._tokenize()
-            self.token = self.iter.next()
-            return self.expression()
-        except:
-            raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping))
-
-    __call__ = parse
-
-def parse(text, **values):
-    """
-    Parse and evaluate a boolean expression.
-    :param text: The expression to parse, as a string.
-    :param values: A dict containing a name to value mapping for identifiers
-                   referenced in *text*.
-    :rtype: the final value of the expression.
-    :raises: :py:exc::ParseError: will be raised if parsing fails.
-    """
-    return ExpressionParser(text, values).parse()
-
-
 ### path normalization
 
 def normalize_path(path):
     """normalize a relative path"""
     if sys.platform.startswith('win'):
         return path.replace('/', os.path.sep)
     return path
 
 def denormalize_path(path):
     """denormalize a relative path"""
     if sys.platform.startswith('win'):
         return path.replace(os.path.sep, '/')
     return path
 
 
-### .ini reader
-
-def read_ini(fp, variables=None, default='DEFAULT', defaults_only=False,
-             comments=';#', separators=('=', ':'),
-             strict=True):
-    """
-    read an .ini file and return a list of [(section, values)]
-    - fp : file pointer or path to read
-    - variables : default set of variables
-    - default : name of the section for the default section
-    - defaults_only : if True, return the default section only
-    - comments : characters that if they start a line denote a comment
-    - separators : strings that denote key, value separation in order
-    - strict : whether to be strict about parsing
-    """
-
-    # variables
-    variables = variables or {}
-    sections = []
-    key = value = None
-    section_names = set()
-    if isinstance(fp, basestring):
-        fp = file(fp)
-
-    # read the lines
-    for (linenum, line) in enumerate(fp.readlines(), start=1):
-
-        stripped = line.strip()
-
-        # ignore blank lines
-        if not stripped:
-            # reset key and value to avoid continuation lines
-            key = value = None
-            continue
-
-        # ignore comment lines
-        if stripped[0] in comments:
-            continue
-
-        # check for a new section
-        if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']':
-            section = stripped[1:-1].strip()
-            key = value = None
-
-            # deal with DEFAULT section
-            if section.lower() == default.lower():
-                if strict:
-                    assert default not in section_names
-                section_names.add(default)
-                current_section = variables
-                continue
-
-            if strict:
-                # make sure this section doesn't already exist
-                assert section not in section_names, "Section '%s' already found in '%s'" % (section, section_names)
-
-            section_names.add(section)
-            current_section = {}
-            sections.append((section, current_section))
-            continue
-
-        # if there aren't any sections yet, something bad happen
-        if not section_names:
-            raise Exception('No sections found')
-
-        # (key, value) pair
-        for separator in separators:
-            if separator in stripped:
-                key, value = stripped.split(separator, 1)
-                key = key.strip()
-                value = value.strip()
-
-                if strict:
-                    # make sure this key isn't already in the section or empty
-                    assert key
-                    if current_section is not variables:
-                        assert key not in current_section
-
-                current_section[key] = value
-                break
-        else:
-            # continuation line ?
-            if line[0].isspace() and key:
-                value = '%s%s%s' % (value, os.linesep, stripped)
-                current_section[key] = value
-            else:
-                # something bad happened!
-                if hasattr(fp, 'name'):
-                    filename = fp.name
-                else:
-                    filename = 'unknown'
-                raise Exception("Error parsing manifest file '%s', line %s" %
-                                (filename, linenum))
-
-    # server-root is a special os path declared relative to the manifest file.
-    # inheritance demands we expand it as absolute
-    if 'server-root' in variables:
-        root = os.path.join(os.path.dirname(fp.name),
-                            variables['server-root'])
-        variables['server-root'] = os.path.abspath(root)
-
-    # return the default section only if requested
-    if defaults_only:
-        return [(default, variables)]
-
-    # interpret the variables
-    def interpret_variables(global_dict, local_dict):
-        variables = global_dict.copy()
-        if 'skip-if' in local_dict and 'skip-if' in variables:
-            local_dict['skip-if'] = "(%s) || (%s)" % (variables['skip-if'].split('#')[0], local_dict['skip-if'].split('#')[0])
-        variables.update(local_dict)
-
-        return variables
-
-    sections = [(i, interpret_variables(variables, j)) for i, j in sections]
-    return sections
-
-
 ### objects for parsing manifests
 
 class ManifestParser(object):
     """read .ini manifests"""
 
     def __init__(self, manifests=(), defaults=None, strict=True):
         self._defaults = defaults or {}
         self._ancestor_defaults = {}
@@ -1178,232 +810,8 @@ class TestManifest(ManifestParser):
             tests = [test for test in tests
                      if not 'disabled' in test]
 
         # return active tests
         return tests
 
     def test_paths(self):
         return [test['path'] for test in self.active_tests()]
-
-
-### command line attributes
-
-class ParserError(Exception):
-  """error for exceptions while parsing the command line"""
-
-def parse_args(_args):
-    """
-    parse and return:
-    --keys=value (or --key value)
-    -tags
-    args
-    """
-
-    # return values
-    _dict = {}
-    tags = []
-    args = []
-
-    # parse the arguments
-    key = None
-    for arg in _args:
-        if arg.startswith('---'):
-            raise ParserError("arguments should start with '-' or '--' only")
-        elif arg.startswith('--'):
-            if key:
-                raise ParserError("Key %s still open" % key)
-            key = arg[2:]
-            if '=' in key:
-                key, value = key.split('=', 1)
-                _dict[key] = value
-                key = None
-                continue
-        elif arg.startswith('-'):
-            if key:
-                raise ParserError("Key %s still open" % key)
-            tags.append(arg[1:])
-            continue
-        else:
-            if key:
-                _dict[key] = arg
-                continue
-            args.append(arg)
-
-    # return values
-    return (_dict, tags, args)
-
-
-### classes for subcommands
-
-class CLICommand(object):
-    usage = '%prog [options] command'
-    def __init__(self, parser):
-      self._parser = parser # master parser
-    def parser(self):
-      return OptionParser(usage=self.usage, description=self.__doc__,
-                          add_help_option=False)
-
-class Copy(CLICommand):
-    usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
-    def __call__(self, options, args):
-      # parse the arguments
-      try:
-        kwargs, tags, args = parse_args(args)
-      except ParserError, e:
-        self._parser.error(e.message)
-
-      # make sure we have some manifests, otherwise it will
-      # be quite boring
-      if not len(args) == 2:
-        HelpCLI(self._parser)(options, ['copy'])
-        return
-
-      # read the manifests
-      # TODO: should probably ensure these exist here
-      manifests = ManifestParser()
-      manifests.read(args[0])
-
-      # print the resultant query
-      manifests.copy(args[1], None, *tags, **kwargs)
-
-
-class CreateCLI(CLICommand):
-    """
-    create a manifest from a list of directories
-    """
-    usage = '%prog [options] create directory <directory> <...>'
-
-    def parser(self):
-        parser = CLICommand.parser(self)
-        parser.add_option('-p', '--pattern', dest='pattern',
-                          help="glob pattern for files")
-        parser.add_option('-i', '--ignore', dest='ignore',
-                          default=[], action='append',
-                          help='directories to ignore')
-        parser.add_option('-w', '--in-place', dest='in_place',
-                          help='Write .ini files in place; filename to write to')
-        return parser
-
-    def __call__(self, _options, args):
-        parser = self.parser()
-        options, args = parser.parse_args(args)
-
-        # need some directories
-        if not len(args):
-            parser.print_usage()
-            return
-
-        # add the directories to the manifest
-        for arg in args:
-            assert os.path.exists(arg)
-            assert os.path.isdir(arg)
-            manifest = convert(args, pattern=options.pattern, ignore=options.ignore,
-                               write=options.in_place)
-        if manifest:
-            print manifest
-
-
-class WriteCLI(CLICommand):
-    """
-    write a manifest based on a query
-    """
-    usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...'
-    def __call__(self, options, args):
-
-        # parse the arguments
-        try:
-            kwargs, tags, args = parse_args(args)
-        except ParserError, e:
-            self._parser.error(e.message)
-
-        # make sure we have some manifests, otherwise it will
-        # be quite boring
-        if not args:
-            HelpCLI(self._parser)(options, ['write'])
-            return
-
-        # read the manifests
-        # TODO: should probably ensure these exist here
-        manifests = ManifestParser()
-        manifests.read(*args)
-
-        # print the resultant query
-        manifests.write(global_tags=tags, global_kwargs=kwargs)
-
-
-class HelpCLI(CLICommand):
-    """
-    get help on a command
-    """
-    usage = '%prog [options] help [command]'
-
-    def __call__(self, options, args):
-        if len(args) == 1 and args[0] in commands:
-            commands[args[0]](self._parser).parser().print_help()
-        else:
-            self._parser.print_help()
-            print '\nCommands:'
-            for command in sorted(commands):
-                print '  %s : %s' % (command, commands[command].__doc__.strip())
-
-class UpdateCLI(CLICommand):
-    """
-    update the tests as listed in a manifest from a directory
-    """
-    usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
-
-    def __call__(self, options, args):
-        # parse the arguments
-        try:
-            kwargs, tags, args = parse_args(args)
-        except ParserError, e:
-            self._parser.error(e.message)
-
-        # make sure we have some manifests, otherwise it will
-        # be quite boring
-        if not len(args) == 2:
-            HelpCLI(self._parser)(options, ['update'])
-            return
-
-        # read the manifests
-        # TODO: should probably ensure these exist here
-        manifests = ManifestParser()
-        manifests.read(args[0])
-
-        # print the resultant query
-        manifests.update(args[1], None, *tags, **kwargs)
-
-
-# command -> class mapping
-commands = { 'create': CreateCLI,
-             'help': HelpCLI,
-             'update': UpdateCLI,
-             'write': WriteCLI }
-
-def main(args=sys.argv[1:]):
-    """console_script entry point"""
-
-    # set up an option parser
-    usage = '%prog [options] [command] ...'
-    description = "%s. Use `help` to display commands" % __doc__.strip()
-    parser = OptionParser(usage=usage, description=description)
-    parser.add_option('-s', '--strict', dest='strict',
-                      action='store_true', default=False,
-                      help='adhere strictly to errors')
-    parser.disable_interspersed_args()
-
-    options, args = parser.parse_args(args)
-
-    if not args:
-        HelpCLI(parser)(options, args)
-        parser.exit()
-
-    # get the command
-    command = args[0]
-    if command not in commands:
-        parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command))
-
-    handler = commands[command](parser)
-    handler(options, args[1:])
-
-if __name__ == '__main__':
-    main()
--- a/testing/mozbase/manifestparser/setup.py
+++ b/testing/mozbase/manifestparser/setup.py
@@ -17,11 +17,11 @@ setup(name=PACKAGE_NAME,
       author_email='tools@lists.mozilla.org',
       url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase',
       license='MPL',
       zip_safe=False,
       packages=['manifestparser'],
       install_requires=[],
       entry_points="""
       [console_scripts]
-      manifestparser = manifestparser.manifestparser:main
+      manifestparser = manifestparser.cli:main
       """,
      )