author | Francesco Pischedda <francesco.pischedda@gmail.com> |
Fri, 30 Sep 2016 16:08:37 +0200 | |
changeset 317796 | f7b6a16eb1f610c2a0291ebcc8474c72b4e993ca |
parent 317795 | 797b8b501271080944a1ea7970484872db49390d |
child 317797 | 869a14571e7e58445b0a33723887cf09de5cbc4a |
push id | 30817 |
push user | cbook@mozilla.com |
push date | Fri, 14 Oct 2016 09:56:08 +0000 |
treeherder | mozilla-central@a71215ad8ab8 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | ahal |
bugs | 1280573 |
milestone | 52.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/testing/mozbase/docs/_static/structured_example.py +++ b/testing/mozbase/docs/_static/structured_example.py @@ -1,51 +1,60 @@ import argparse import sys import traceback import types from mozlog import commandline, get_default_logger + class TestAssertion(Exception): pass + def assert_equals(a, b): if a != b: raise TestAssertion("%r not equal to %r" % (a, b)) + def expected(status): def inner(f): def test_func(): f() test_func.__name__ = f.__name__ test_func._expected = status return test_func return inner + def test_that_passes(): assert_equals(1, int("1")) + def test_that_fails(): assert_equals(1, int("2")) + def test_that_has_an_error(): assert_equals(2, 1 + "1") + @expected("FAIL") def test_expected_fail(): assert_equals(2 + 2, 5) + class TestRunner(object): + def __init__(self): self.logger = get_default_logger(component='TestRunner') def gather_tests(self): for item in globals().itervalues(): - if type(item) == types.FunctionType and item.__name__.startswith("test_"): + if isinstance(item, types.FunctionType) and item.__name__.startswith("test_"): yield item.__name__, item def run(self): tests = list(self.gather_tests()) self.logger.suite_start(tests=[name for name, func in tests]) self.logger.info("Running tests") for name, func in tests: @@ -64,20 +73,22 @@ class TestRunner(object): message = e.message except: status = "ERROR" message = traceback.format_exc() else: status = "PASS" self.logger.test_end(name, status=status, expected=expected, message=message) + def get_parser(): parser = argparse.ArgumentParser() return parser + def main(): parser = get_parser() commandline.add_logging_group(parser) args = parser.parse_args() logger = commandline.setup_logging("structured-example", args, {"raw": sys.stdout})
--- a/testing/mozbase/docs/conf.py +++ b/testing/mozbase/docs/conf.py @@ -6,46 +6,47 @@ # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. here = os.path.dirname(os.path.abspath(__file__)) parent = os.path.dirname(here) for item in os.listdir(parent): path = os.path.join(parent, item) if (not os.path.isdir(path)) or (not os.path.exists(os.path.join(path, 'setup.py'))): continue sys.path.insert(0, path) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.viewcode'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'MozBase' copyright = u'2012, Mozilla Automation and Tools team' @@ -55,47 +56,47 @@ copyright = u'2012, Mozilla Automation a # # The short X.Y version. version = '1' # The full version, including alpha/beta/rc tags. release = '1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'default' on_rtd = os.environ.get('READTHEDOCS', None) == 'True' @@ -106,152 +107,152 @@ if not on_rtd: html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] except ImportError: pass # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # "<project> v<release> documentation". html_title = "mozbase documentation" # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a <link> tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'MozBasedoc' # -- Options for LaTeX output -------------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'MozBase.tex', u'MozBase Documentation', - u'Mozilla Automation and Tools team', 'manual'), + ('index', 'MozBase.tex', u'MozBase Documentation', + u'Mozilla Automation and Tools team', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'mozbase', u'MozBase Documentation', [u'Mozilla Automation and Tools team'], 1) ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'MozBase', u'MozBase Documentation', - u'Mozilla Automation and Tools team', 'MozBase', 'One line description of project.', - 'Miscellaneous'), + ('index', 'MozBase', u'MozBase Documentation', + u'Mozilla Automation and Tools team', 'MozBase', 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote'
--- a/testing/mozbase/manifestparser/manifestparser/__init__.py +++ b/testing/mozbase/manifestparser/manifestparser/__init__.py @@ -1,7 +1,8 @@ +# flake8: noqa # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. from .manifestparser import * from .expression import * from .ini import *
--- a/testing/mozbase/manifestparser/manifestparser/cli.py +++ b/testing/mozbase/manifestparser/manifestparser/cli.py @@ -13,17 +13,18 @@ import sys from .manifestparser import ( convert, ManifestParser, ) class ParserError(Exception): - """error for exceptions while parsing the command line""" + """error for exceptions while parsing the command line""" + def parse_args(_args): """ parse and return: --keys=value (or --key value) -tags args """ @@ -56,46 +57,51 @@ def parse_args(_args): if key: _dict[key] = arg continue args.append(arg) # return values return (_dict, tags, args) + class CLICommand(object): usage = '%prog [options] command' + def __init__(self, parser): - self._parser = parser # master parser + self._parser = parser # master parser + def parser(self): - return OptionParser(usage=self.usage, description=self.__doc__, - add_help_option=False) + return OptionParser(usage=self.usage, description=self.__doc__, + add_help_option=False) + class Copy(CLICommand): usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...' + def __call__(self, options, args): - # parse the arguments - try: - kwargs, tags, args = parse_args(args) - except ParserError, e: - self._parser.error(e.message) + # parse the arguments + try: + kwargs, tags, args = parse_args(args) + except ParserError, e: + self._parser.error(e.message) - # make sure we have some manifests, otherwise it will - # be quite boring - if not len(args) == 2: - HelpCLI(self._parser)(options, ['copy']) - return + # make sure we have some manifests, otherwise it will + # be quite boring + if not len(args) == 2: + HelpCLI(self._parser)(options, ['copy']) + return - # read the manifests - # TODO: should probably ensure these exist here - manifests = ManifestParser() - manifests.read(args[0]) + # read the manifests + # TODO: should probably ensure these exist here + manifests = ManifestParser() + manifests.read(args[0]) - # print the resultant query - manifests.copy(args[1], None, *tags, **kwargs) + # print the resultant query + manifests.copy(args[1], None, *tags, **kwargs) class CreateCLI(CLICommand): """ create a manifest from a list of directories """ usage = '%prog [options] create directory <directory> <...>' @@ -129,16 +135,17 @@ class CreateCLI(CLICommand): print manifest class WriteCLI(CLICommand): """ write a manifest based on a query """ usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...' + def __call__(self, options, args): # parse the arguments try: kwargs, tags, args = parse_args(args) except ParserError, e: self._parser.error(e.message) @@ -152,32 +159,32 @@ class WriteCLI(CLICommand): # TODO: should probably ensure these exist here manifests = ManifestParser() manifests.read(*args) # print the resultant query manifests.write(global_tags=tags, global_kwargs=kwargs) - class HelpCLI(CLICommand): """ get help on a command """ usage = '%prog [options] help [command]' def __call__(self, options, args): if len(args) == 1 and args[0] in commands: commands[args[0]](self._parser).parser().print_help() else: self._parser.print_help() print '\nCommands:' for command in sorted(commands): print ' %s : %s' % (command, commands[command].__doc__.strip()) + class UpdateCLI(CLICommand): """ update the tests as listed in a manifest from a directory """ usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...' def __call__(self, options, args): # parse the arguments @@ -197,20 +204,21 @@ class UpdateCLI(CLICommand): manifests = ManifestParser() manifests.read(args[0]) # print the resultant query manifests.update(args[1], None, *tags, **kwargs) # command -> class mapping -commands = { 'create': CreateCLI, - 'help': HelpCLI, - 'update': UpdateCLI, - 'write': WriteCLI } +commands = {'create': CreateCLI, + 'help': HelpCLI, + 'update': UpdateCLI, + 'write': WriteCLI} + def main(args=sys.argv[1:]): """console_script entry point""" # set up an option parser usage = '%prog [options] [command] ...' description = "%s. Use `help` to display commands" % __doc__.strip() parser = OptionParser(usage=usage, description=description) @@ -223,15 +231,16 @@ def main(args=sys.argv[1:]): if not args: HelpCLI(parser)(options, args) parser.exit() # get the command command = args[0] if command not in commands: - parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command)) + parser.error("Command must be one of %s (you gave '%s')" % + (', '.join(sorted(commands.keys())), command)) handler = commands[command](parser) handler(options, args[1:]) if __name__ == '__main__': main()
--- a/testing/mozbase/manifestparser/manifestparser/expression.py +++ b/testing/mozbase/manifestparser/manifestparser/expression.py @@ -40,119 +40,155 @@ import traceback # argument. # Glossary (see above URL for details): # - nud: null denotation # - led: left detonation # - lbp: left binding power # - rbp: right binding power + class ident_token(object): + def __init__(self, scanner, value): self.value = value + def nud(self, parser): # identifiers take their value from the value mappings passed # to the parser return parser.value(self.value) + class literal_token(object): + def __init__(self, scanner, value): self.value = value + def nud(self, parser): return self.value + class eq_op_token(object): "==" + def led(self, parser, left): return left == parser.expression(self.lbp) + class neq_op_token(object): "!=" + def led(self, parser, left): return left != parser.expression(self.lbp) + class lt_op_token(object): "<" + def led(self, parser, left): return left < parser.expression(self.lbp) + class gt_op_token(object): ">" + def led(self, parser, left): return left > parser.expression(self.lbp) + class le_op_token(object): "<=" + def led(self, parser, left): return left <= parser.expression(self.lbp) + class ge_op_token(object): ">=" + def led(self, parser, left): return left >= parser.expression(self.lbp) + class not_op_token(object): "!" + def nud(self, parser): return not parser.expression(100) + class and_op_token(object): "&&" + def led(self, parser, left): right = parser.expression(self.lbp) return left and right + class or_op_token(object): "||" + def led(self, parser, left): right = parser.expression(self.lbp) return left or right + class lparen_token(object): "(" + def nud(self, parser): expr = parser.expression() parser.advance(rparen_token) return expr + class rparen_token(object): ")" + class end_token(object): """always ends parsing""" -### derived literal tokens +# derived literal tokens + class bool_token(literal_token): + def __init__(self, scanner, value): - value = {'true':True, 'false':False}[value] + value = {'true': True, 'false': False}[value] literal_token.__init__(self, scanner, value) + class int_token(literal_token): + def __init__(self, scanner, value): literal_token.__init__(self, scanner, int(value)) + class string_token(literal_token): + def __init__(self, scanner, value): literal_token.__init__(self, scanner, value[1:-1]) precedence = [(end_token, rparen_token), (or_op_token,), (and_op_token,), (lt_op_token, gt_op_token, le_op_token, ge_op_token, eq_op_token, neq_op_token), (lparen_token,), ] for index, rank in enumerate(precedence): for token in rank: - token.lbp = index # lbp = lowest left binding power + token.lbp = index # lbp = lowest left binding power + class ParseError(Exception): """error parsing conditional expression""" + class ExpressionParser(object): """ A parser for a simple expression language. The expression language can be described as follows:: EXPRESSION ::= LITERAL | '(' EXPRESSION ')' | '!' EXPRESSION | EXPRESSION OP EXPRESSION OP ::= '==' | '!=' | '<' | '>' | '<=' | '>=' | '&&' | '||' @@ -210,17 +246,17 @@ class ExpressionParser(object): (r">=", ge_op_token()), (r"<", lt_op_token()), (r">", gt_op_token()), (r"\|\|", or_op_token()), (r"!", not_op_token()), (r"&&", and_op_token()), (r"\(", lparen_token()), (r"\)", rparen_token()), - (r"\s+", None), # skip whitespace + (r"\s+", None), # skip whitespace ]) tokens, remainder = ExpressionParser.scanner.scan(self.text) for t in tokens: yield t yield end_token() def value(self, ident): """ @@ -233,17 +269,17 @@ class ExpressionParser(object): return self.valuemapping.get(ident, None) def advance(self, expected): """ Assert that the next token is an instance of |expected|, and advance to the next token. """ if not isinstance(self.token, expected): - raise Exception, "Unexpected token!" + raise Exception("Unexpected token!") self.token = self.iter.next() def expression(self, rbp=0): """ Parse and return the value of an expression until a token with right binding power greater than rbp is encountered. """ t = self.token @@ -263,17 +299,20 @@ class ExpressionParser(object): """ try: self.iter = self._tokenize() self.token = self.iter.next() return self.expression() except: extype, ex, tb = sys.exc_info() formatted = ''.join(traceback.format_exception_only(extype, ex)) - raise ParseError("could not parse: %s\nexception: %svariables: %s" % (self.text, formatted, self.valuemapping)), None, tb + raise ParseError("could not parse: " + "%s\nexception: %svariables: %s" % (self.text, + formatted, + self.valuemapping)), None, tb __call__ = parse def parse(text, **values): """ Parse and evaluate a boolean expression. :param text: The expression to parse, as a string.
--- a/testing/mozbase/manifestparser/manifestparser/filters.py +++ b/testing/mozbase/manifestparser/manifestparser/filters.py @@ -112,16 +112,17 @@ class subsuite(InstanceFilter): subsuite = foo,condition where 'foo' is the subsuite name, and 'condition' is the same type of condition used for skip-if. If the condition doesn't evaluate to true, the subsuite designation will be removed from the test. :param name: The name of the subsuite to run (default None) """ + def __init__(self, name=None): InstanceFilter.__init__(self, name=name) self.name = name def __call__(self, tests, values): # Look for conditional subsuites, and replace them with the subsuite # itself (if the condition is true), or nothing. for test in tests: @@ -222,17 +223,17 @@ class chunk_by_dir(InstanceFilter): ordered_dirs = [] for test in tests: path = test['relpath'] if path.startswith(os.sep): path = path[1:] dirs = path.split(os.sep) - dirs = dirs[:min(self.depth, len(dirs)-1)] + dirs = dirs[:min(self.depth, len(dirs) - 1)] path = os.sep.join(dirs) # don't count directories that only have disabled tests in them, # but still yield disabled tests that are alongside enabled tests if path not in ordered_dirs and 'disabled' not in test: ordered_dirs.append(path) tests_by_dir[path].append(test) @@ -298,17 +299,17 @@ class chunk_by_runtime(InstanceFilter): for runtime, batch in tests_by_manifest: # sort first by runtime, then by number of tests in case of a tie. # This guarantees the chunk with the fastest runtime will always # get the next batch of tests. tests_by_chunk.sort(key=lambda x: (x[0], len(x[1]))) tests_by_chunk[0][0] += runtime tests_by_chunk[0][1].extend(batch) - return (t for t in tests_by_chunk[self.this_chunk-1][1]) + return (t for t in tests_by_chunk[self.this_chunk - 1][1]) class tags(InstanceFilter): """ Removes tests that don't contain any of the given tags. This overrides InstanceFilter's __eq__ method, so multiple instances can be added. Multiple tag filters is equivalent to joining tags with the AND operator.
--- a/testing/mozbase/manifestparser/manifestparser/ini.py +++ b/testing/mozbase/manifestparser/manifestparser/ini.py @@ -1,15 +1,16 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. +import os + __all__ = ['read_ini'] -import os def read_ini(fp, variables=None, default='DEFAULT', defaults_only=False, comments=';#', separators=('=', ':'), strict=True): """ read an .ini file and return a list of [(section, values)] - fp : file pointer or path to read - variables : default set of variables @@ -53,17 +54,18 @@ def read_ini(fp, variables=None, default if strict: assert default not in section_names section_names.add(default) current_section = variables continue if strict: # make sure this section doesn't already exist - assert section not in section_names, "Section '%s' already found in '%s'" % (section, section_names) + assert section not in section_names, "Section '%s' already found in '%s'" % ( + section, section_names) section_names.add(section) current_section = {} sections.append((section, current_section)) continue # if there aren't any sections yet, something bad happen if not section_names: @@ -114,15 +116,16 @@ def read_ini(fp, variables=None, default variables = global_dict.copy() # These variables are combinable when they appear both in default # and per-entry. for field_name, pattern in (('skip-if', '(%s) || (%s)'), ('support-files', '%s %s')): local_value, global_value = local_dict.get(field_name), variables.get(field_name) if local_value and global_value: - local_dict[field_name] = pattern % (global_value.split('#')[0], local_value.split('#')[0]) + local_dict[field_name] = pattern % ( + global_value.split('#')[0], local_value.split('#')[0]) variables.update(local_dict) return variables sections = [(i, interpret_variables(variables, j)) for i, j in sections] return sections
--- a/testing/mozbase/manifestparser/manifestparser/manifestparser.py +++ b/testing/mozbase/manifestparser/manifestparser/manifestparser.py @@ -1,50 +1,51 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -__all__ = ['ManifestParser', 'TestManifest', 'convert'] - from StringIO import StringIO import json import fnmatch import os import shutil import sys import types from .ini import read_ini from .filters import ( DEFAULT_FILTERS, enabled, exists as _exists, filterlist, ) +__all__ = ['ManifestParser', 'TestManifest', 'convert'] + relpath = os.path.relpath string = (basestring,) -### path normalization +# path normalization def normalize_path(path): """normalize a relative path""" if sys.platform.startswith('win'): return path.replace('/', os.path.sep) return path + def denormalize_path(path): """denormalize a relative path""" if sys.platform.startswith('win'): return path.replace(os.path.sep, '/') return path -### objects for parsing manifests +# objects for parsing manifests class ManifestParser(object): """read .ini manifests""" def __init__(self, manifests=(), defaults=None, strict=True, rootdir=None, finder=None): """Creates a ManifestParser from the given manifest files. @@ -77,17 +78,17 @@ class ManifestParser(object): if manifests: self.read(*manifests) def path_exists(self, path): if self.finder: return self.finder.get(path) is not None return os.path.exists(path) - ### methods for reading manifests + # methods for reading manifests def _read(self, root, filename, defaults, defaults_only=False, parentmanifest=None): """ Internal recursive method for reading and parsing manifests. Stores all found tests in self.tests :param root: The base path :param filename: File object or string path for the base manifest file :param defaults: Options that apply to all items @@ -181,17 +182,17 @@ class ManifestParser(object): test['name'] = section # Will be None if the manifest being read is a file-like object. test['manifest'] = filename # determine the path path = test.get('path', section) _relpath = path - if '://' not in path: # don't futz with URLs + if '://' not in path: # don't futz with URLs path = normalize_path(path) if here and not os.path.isabs(path): # Profiling indicates 25% of manifest parsing is spent # in this call to normpath, but almost all calls return # their argument unmodified, so we avoid the call if # '..' if not present in the path. path = os.path.join(here, path) if '..' in path: @@ -255,27 +256,26 @@ class ManifestParser(object): # process each file for filename in filenames: # set the per file defaults defaults = _defaults.copy() here = None if isinstance(filename, string): here = os.path.dirname(os.path.abspath(filename)) - defaults['here'] = here # directory of master .ini file + defaults['here'] = here # directory of master .ini file if self.rootdir is None: # set the root directory # == the directory of the first manifest given self.rootdir = here self._read(here, filename, defaults) - - ### methods for querying manifests + # methods for querying manifests def query(self, *checks, **kw): """ general query function for tests - checks : callable conditions to test if the test fulfills the query """ tests = kw.get('tests', None) if tests is None: @@ -299,24 +299,28 @@ class ManifestParser(object): # fix up tags if tags: tags = set(tags) else: tags = set() # make some check functions if inverse: - has_tags = lambda test: not tags.intersection(test.keys()) + def has_tags(test): + return not tags.intersection(test.keys()) + def dict_query(test): for key, value in kwargs.items(): if test.get(key) == value: return False return True else: - has_tags = lambda test: tags.issubset(test.keys()) + def has_tags(test): + return tags.issubset(test.keys()) + def dict_query(test): for key, value in kwargs.items(): if test.get(key) != value: return False return True # query the tests tests = self.query(has_tags, dict_query, tests=tests) @@ -344,18 +348,17 @@ class ManifestParser(object): continue if manifest not in manifests: manifests.append(manifest) return manifests def paths(self): return [i['path'] for i in self.tests] - - ### methods for auditing + # methods for auditing def missing(self, tests=None): """ return list of tests that do not exist on the filesystem """ if tests is None: tests = self.tests existing = list(_exists(tests, {})) @@ -365,17 +368,17 @@ class ManifestParser(object): missing = self.missing(tests=tests) if missing: missing_paths = [test['path'] for test in missing] if self.strict: raise IOError("Strict mode enabled, test paths must exist. " "The following test(s) are missing: %s" % json.dumps(missing_paths, indent=2)) print >> sys.stderr, "Warning: The following test(s) are missing: %s" % \ - json.dumps(missing_paths, indent=2) + json.dumps(missing_paths, indent=2) return missing def verifyDirectory(self, directories, pattern=None, extensions=None): """ checks what is on the filesystem vs what is in a manifest returns a 2-tuple of sets: (missing_from_filesystem, missing_from_manifest) """ @@ -399,18 +402,17 @@ class ManifestParser(object): files.update([os.path.join(dirpath, filename) for filename in filenames]) paths = set(self.paths()) missing_from_filesystem = paths.difference(files) missing_from_manifest = files.difference(paths) return (missing_from_filesystem, missing_from_manifest) - - ### methods for output + # methods for output def write(self, fp=sys.stdout, rootdir=None, global_tags=None, global_kwargs=None, local_tags=None, local_kwargs=None): """ write a manifest given a query global and local options will be munged to do the query globals will be written to the top of the file @@ -449,17 +451,17 @@ class ManifestParser(object): print >> fp, '[DEFAULT]' for tag in global_tags: print >> fp, '%s =' % tag for key, value in global_kwargs.items(): print >> fp, '%s = %s' % (key, value) print >> fp for test in tests: - test = test.copy() # don't overwrite + test = test.copy() # don't overwrite path = test['name'] if not os.path.isabs(path): path = test['path'] if self.rootdir: path = relpath(test['path'], self.rootdir) path = denormalize_path(path) print >> fp, '[%s]' % path @@ -504,17 +506,17 @@ class ManifestParser(object): os.path.makedirs(directory) else: # sanity check assert os.path.isdir(directory) # tests to copy tests = self.get(tags=tags, **kwargs) if not tests: - return # nothing to do! + return # nothing to do! # root directory if rootdir is None: rootdir = self.rootdir # copy the manifests + tests manifests = [relpath(manifest, rootdir) for manifest in self.manifests()] for manifest in manifests: @@ -562,42 +564,45 @@ class ManifestParser(object): message = "Missing test: '%s' does not exist!" if self.strict: raise IOError(message) print >> sys.stderr, message + " Skipping." continue destination = os.path.join(rootdir, _relpath) shutil.copy(source, destination) - ### directory importers + # directory importers @classmethod def _walk_directories(cls, directories, callback, pattern=None, ignore=()): """ internal function to import directories """ if isinstance(pattern, basestring): patterns = [pattern] else: patterns = pattern ignore = set(ignore) if not patterns: - accept_filename = lambda filename: True + def accept_filename(filename): + return True else: def accept_filename(filename): for pattern in patterns: if fnmatch.fnmatch(filename, pattern): return True if not ignore: - accept_dirname = lambda dirname: True + def accept_dirname(dirname): + return True else: - accept_dirname = lambda dirname: dirname not in ignore + def accept_dirname(dirname): + return dirname not in ignore rootdirectories = directories[:] seen_directories = set() for rootdirectory in rootdirectories: # let's recurse directories using list directories = [os.path.realpath(rootdirectory)] while directories: directory = directories.pop(0) @@ -627,22 +632,22 @@ class ManifestParser(object): # this subdir is added for recursion directories.insert(0, path) # here we got all subdirs and files filtered, we can # call the callback function if directory is not empty if subdirs or files: callback(rootdirectory, directory, subdirs, files) - @classmethod - def populate_directory_manifests(cls, directories, filename, pattern=None, ignore=(), overwrite=False): + def populate_directory_manifests(cls, directories, filename, pattern=None, ignore=(), + overwrite=False): """ - walks directories and writes manifests of name `filename` in-place; returns `cls` instance populated - with the given manifests + walks directories and writes manifests of name `filename` in-place; + returns `cls` instance populated with the given manifests filename -- filename of manifests to write pattern -- shell pattern (glob) or patterns of filenames to match ignore -- directory names to ignore overwrite -- whether to overwrite existing files of given name """ manifest_dict = {} @@ -687,20 +692,19 @@ class ManifestParser(object): pattern -- shell pattern (glob) or patterns of filenames to match ignore -- directory names to ignore write -- filename or file-like object of manifests to write; if `None` then a StringIO instance will be created relative_to -- write paths relative to this path; if false then the paths are absolute """ - # determine output - opened_manifest_file = None # name of opened manifest file - absolute = not relative_to # whether to output absolute path names as names + opened_manifest_file = None # name of opened manifest file + absolute = not relative_to # whether to output absolute path names as names if isinstance(write, string): opened_manifest_file = write write = file(write, 'w') if write is None: write = StringIO() # walk the directories, generating manifests def callback(directory, dirpath, dirnames, filenames): @@ -713,33 +717,31 @@ class ManifestParser(object): if filename != opened_manifest_file] # normalize paths if not absolute and relative_to: filenames = [relpath(filename, relative_to) for filename in filenames] # write to manifest print >> write, '\n'.join(['[%s]' % denormalize_path(filename) - for filename in filenames]) - + for filename in filenames]) cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore) if opened_manifest_file: # close file write.close() manifests = [opened_manifest_file] else: # manifests/write is a file-like object; # rewind buffer write.flush() write.seek(0) manifests = [write] - # make a ManifestParser instance return cls(manifests=manifests) convert = ManifestParser.from_directories class TestManifest(ManifestParser): """ @@ -757,17 +759,17 @@ class TestManifest(ManifestParser): Run all applied filters on the set of tests. :param exists: filter out non-existing tests (default True) :param disabled: whether to return disabled tests (default True) :param values: keys and values to filter on (e.g. `os = linux mac`) :param filters: list of filters to apply to the tests :returns: list of test objects that were not filtered out """ - tests = [i.copy() for i in self.tests] # shallow copy + tests = [i.copy() for i in self.tests] # shallow copy # mark all tests as passing for test in tests: test['expected'] = test.get('expected', 'pass') # make a copy so original doesn't get modified fltrs = self.filters[:] if exists:
--- a/testing/mozbase/manifestparser/setup.py +++ b/testing/mozbase/manifestparser/setup.py @@ -6,22 +6,22 @@ from setuptools import setup PACKAGE_NAME = "manifestparser" PACKAGE_VERSION = '1.1' setup(name=PACKAGE_NAME, version=PACKAGE_VERSION, description="Library to create and manage test manifests", long_description="see http://mozbase.readthedocs.org/", - classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla manifests', author='Mozilla Automation and Testing Team', author_email='tools@lists.mozilla.org', url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', license='MPL', zip_safe=False, packages=['manifestparser'], install_requires=[], entry_points=""" [console_scripts] manifestparser = manifestparser.cli:main """, - ) + )
--- a/testing/mozbase/manifestparser/tests/test_chunking.py +++ b/testing/mozbase/manifestparser/tests/test_chunking.py @@ -28,17 +28,17 @@ class ChunkBySlice(TestCase): return tests def run_all_combos(self, num_tests, disabled=None): tests = self.generate_tests(num_tests, disabled=disabled) for total in range(1, num_tests + 1): res = [] res_disabled = [] - for chunk in range(1, total+1): + for chunk in range(1, total + 1): f = chunk_by_slice(chunk, total) res.append(list(f(tests, {}))) if disabled: f.disabled = True res_disabled.append(list(f(tests, {}))) lengths = [len([t for t in c if 'disabled' not in t]) for c in res] # the chunk with the most tests should have at most one more test @@ -91,30 +91,30 @@ class ChunkByDir(TestCase): name = 'test%i' % i test = {'name': name, 'relpath': os.path.join(d, name)} yield test def run_all_combos(self, dirs): tests = list(self.generate_tests(dirs)) - deepest = max(len(t['relpath'].split(os.sep))-1 for t in tests) - for depth in range(1, deepest+1): + deepest = max(len(t['relpath'].split(os.sep)) - 1 for t in tests) + for depth in range(1, deepest + 1): def num_groups(tests): unique = set() for p in [t['relpath'] for t in tests]: p = p.split(os.sep) - p = p[:min(depth, len(p)-1)] + p = p[:min(depth, len(p) - 1)] unique.add(os.sep.join(p)) return len(unique) - for total in range(1, num_groups(tests)+1): + for total in range(1, num_groups(tests) + 1): res = [] - for this in range(1, total+1): + for this in range(1, total + 1): f = chunk_by_dir(this, total, depth) res.append(list(f(tests, {}))) lengths = map(num_groups, res) # the chunk with the most dirs should have at most one more # dir than the chunk with the least dirs self.assertLessEqual(max(lengths) - min(lengths), 1) @@ -203,36 +203,36 @@ class ChunkByRuntime(TestCase): chunks = [[] for i in range(total)] d = 1 # direction i = 0 for runtime, batch in tests_by_manifest: chunks[i].extend(batch) # "draft" style (last pick goes first in the next round) - if (i == 0 and d == -1) or (i == total-1 and d == 1): + if (i == 0 and d == -1) or (i == total - 1 and d == 1): d = -d else: i += d # make sure this test algorithm is valid all_chunks = list(chain.from_iterable(chunks)) self.assertEqual(len(all_chunks), len(tests)) for t in tests: self.assertIn(t, all_chunks) return chunks def run_all_combos(self, dirs): tests = list(self.generate_tests(dirs)) runtimes = self.get_runtimes(tests) - for total in range(1, len(dirs)+1): + for total in range(1, len(dirs) + 1): chunks = [] - for this in range(1, total+1): + for this in range(1, total + 1): f = chunk_by_runtime(this, total, runtimes) ret = list(f(tests, {})) chunks.append(ret) # chunk_by_runtime will mess up order, but chained chunks should # contain all of the original tests and be the same length all_chunks = list(chain.from_iterable(chunks)) self.assertEqual(len(all_chunks), len(tests))
--- a/testing/mozbase/manifestparser/tests/test_convert_directory.py +++ b/testing/mozbase/manifestparser/tests/test_convert_directory.py @@ -15,22 +15,25 @@ from manifestparser import ManifestParse here = os.path.dirname(os.path.abspath(__file__)) # In some cases tempfile.mkdtemp() may returns a path which contains # symlinks. Some tests here will then break, as the manifestparser.convert # function returns paths that does not contains symlinks. # # Workaround is to use the following function, if absolute path of temp dir # must be compared. + + def create_realpath_tempdir(): """ Create a tempdir without symlinks. """ return os.path.realpath(tempfile.mkdtemp()) + class TestDirectoryConversion(unittest.TestCase): """test conversion of a directory tree to a manifest structure""" def create_stub(self, directory=None): """stub out a directory with files in it""" files = ('foo', 'bar', 'fleem') if directory is None: @@ -51,34 +54,34 @@ class TestDirectoryConversion(unittest.T # create a stub directory stub = self.create_stub() try: stub = stub.replace(os.path.sep, "/") self.assertTrue(os.path.exists(stub) and os.path.isdir(stub)) # Make a manifest for it manifest = convert([stub]) - self.assertEqual(str(manifest), -"""[%(stub)s/bar] + out_tmpl = """[%(stub)s/bar] subsuite = [%(stub)s/fleem] subsuite = [%(stub)s/foo] subsuite = [%(stub)s/subdir/subfile] subsuite = -""" % dict(stub=stub)) +""" # noqa + self.assertEqual(str(manifest), out_tmpl % dict(stub=stub)) except: raise finally: - shutil.rmtree(stub) # cleanup + shutil.rmtree(stub) # cleanup def test_convert_directory_manifests_in_place(self): """ keep the manifests in place """ stub = self.create_stub() try: @@ -98,17 +101,18 @@ subsuite = finally: shutil.rmtree(stub) def test_manifest_ignore(self): """test manifest `ignore` parameter for ignoring directories""" stub = self.create_stub() try: - ManifestParser.populate_directory_manifests([stub], filename='manifest.ini', ignore=('subdir',)) + ManifestParser.populate_directory_manifests( + [stub], filename='manifest.ini', ignore=('subdir',)) parser = ManifestParser() parser.read(os.path.join(stub, 'manifest.ini')) self.assertEqual([i['name'] for i in parser.tests], ['bar', 'fleem', 'foo']) self.assertFalse(os.path.exists(os.path.join(stub, 'subdir', 'manifest.ini'))) except: raise finally: @@ -157,25 +161,25 @@ subsuite = paths = [str(i) for i in range(10)] self.assertEqual([i['name'] for i in manifest.missing()], paths) # But then we copy one over: self.assertEqual(manifest.get('name', name='1'), ['1']) manifest.update(tempdir, name='1') self.assertEqual(sorted(os.listdir(newtempdir)), - ['1', 'manifest.ini']) + ['1', 'manifest.ini']) # Update that one file and copy all the "tests": file(os.path.join(tempdir, '1'), 'w').write('secret door') manifest.update(tempdir) self.assertEqual(sorted(os.listdir(newtempdir)), - ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'manifest.ini']) + ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'manifest.ini']) self.assertEqual(file(os.path.join(newtempdir, '1')).read().strip(), - 'secret door') + 'secret door') # clean up: shutil.rmtree(tempdir) shutil.rmtree(newtempdir) if __name__ == '__main__': unittest.main()
--- a/testing/mozbase/manifestparser/tests/test_convert_symlinks.py +++ b/testing/mozbase/manifestparser/tests/test_convert_symlinks.py @@ -6,16 +6,17 @@ import os import shutil import tempfile import unittest from manifestparser import convert, ManifestParser + class TestSymlinkConversion(unittest.TestCase): """ test conversion of a directory tree with symlinks to a manifest structure """ def create_stub(self, directory=None): """stub out a directory with files in it""" @@ -120,16 +121,17 @@ class TestSymlinkConversion(unittest.Tes os.symlink(os.path.join('..', 'dir2'), os.path.join(workspace, 'dir1', 'ldir2')) # create one file in each dir open(os.path.join(workspace, 'dir1', 'f1.txt'), 'a').close() open(os.path.join(workspace, 'dir1', 'ldir2', 'f2.txt'), 'a').close() data = [] + def callback(rootdirectory, directory, subdirs, files): for f in files: data.append(f) ManifestParser._walk_directories([workspace], callback) self.assertEqual(sorted(data), ['f1.txt', 'f2.txt'])
--- a/testing/mozbase/manifestparser/tests/test_default_overrides.py +++ b/testing/mozbase/manifestparser/tests/test_default_overrides.py @@ -5,38 +5,40 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. import os import unittest from manifestparser import ManifestParser here = os.path.dirname(os.path.abspath(__file__)) + class TestDefaultSkipif(unittest.TestCase): """Tests applying a skip-if condition in [DEFAULT] and || with the value for the test""" - def test_defaults(self): default = os.path.join(here, 'default-skipif.ini') parser = ManifestParser(manifests=(default,)) for test in parser.tests: if test['name'] == 'test1': self.assertEqual(test['skip-if'], "(os == 'win' && debug ) || (debug)") elif test['name'] == 'test2': self.assertEqual(test['skip-if'], "(os == 'win' && debug ) || (os == 'linux')") elif test['name'] == 'test3': self.assertEqual(test['skip-if'], "(os == 'win' && debug ) || (os == 'win')") elif test['name'] == 'test4': - self.assertEqual(test['skip-if'], "(os == 'win' && debug ) || (os == 'win' && debug)") + self.assertEqual( + test['skip-if'], "(os == 'win' && debug ) || (os == 'win' && debug)") elif test['name'] == 'test5': self.assertEqual(test['skip-if'], "os == 'win' && debug # a pesky comment") elif test['name'] == 'test6': self.assertEqual(test['skip-if'], "(os == 'win' && debug ) || (debug )") + class TestDefaultSupportFiles(unittest.TestCase): """Tests combining support-files field in [DEFAULT] with the value for a test""" def test_defaults(self): default = os.path.join(here, 'default-suppfiles.ini') parser = ManifestParser(manifests=(default,)) expected_supp_files = {
--- a/testing/mozbase/manifestparser/tests/test_expressionparser.py +++ b/testing/mozbase/manifestparser/tests/test_expressionparser.py @@ -1,13 +1,14 @@ #!/usr/bin/env python import unittest from manifestparser import parse + class ExpressionParserTest(unittest.TestCase): """Test the conditional expression parser.""" def test_basic(self): self.assertEqual(parse("1"), 1) self.assertEqual(parse("100"), 100) self.assertEqual(parse("true"), True) @@ -59,17 +60,16 @@ class ExpressionParserTest(unittest.Test self.assertTrue(parse("(true || false)")) self.assertTrue(parse("(true && true || false)")) self.assertFalse(parse("(true || false) && false")) self.assertTrue(parse("(true || false) && true")) self.assertTrue(parse("true && (true || false)")) self.assertTrue(parse("true && (true || false)")) self.assertTrue(parse("(true && false) || (true && (true || false))")) - def test_comments(self): # comments in expressions work accidentally, via an implementation # detail - the '#' character doesn't match any of the regular # expressions we specify as tokens, and thus are ignored. # However, having explicit tests for them means that should the # implementation ever change, comments continue to work, even if that # means a new implementation must handle them explicitly. self.assertTrue(parse("true == true # it does!"))
--- a/testing/mozbase/manifestparser/tests/test_filters.py +++ b/testing/mozbase/manifestparser/tests/test_filters.py @@ -1,9 +1,10 @@ #!/usr/bin/env python +# flake8: noqa from copy import deepcopy import os import unittest from manifestparser.filters import ( subsuite, tags, @@ -139,17 +140,17 @@ class BuiltinFilters(unittest.TestCase): def test_subsuite(self): sub1 = subsuite() sub2 = subsuite('baz') tests = deepcopy(self.tests) tests = list(sub1(tests, {})) self.assertNotIn(self.tests[5], tests) - self.assertEquals(len(tests), len(self.tests)-1) + self.assertEquals(len(tests), len(self.tests) - 1) tests = deepcopy(self.tests) tests = list(sub2(tests, {})) self.assertEquals(len(tests), 1) self.assertIn(self.tests[5], tests) def test_subsuite_condition(self): sub1 = subsuite()
--- a/testing/mozbase/manifestparser/tests/test_manifestparser.py +++ b/testing/mozbase/manifestparser/tests/test_manifestparser.py @@ -8,16 +8,17 @@ import os import shutil import tempfile import unittest from manifestparser import ManifestParser from StringIO import StringIO here = os.path.dirname(os.path.abspath(__file__)) + class TestManifestParser(unittest.TestCase): """ Test the manifest parser You must have manifestparser installed before running these tests. Run ``python manifestparser.py setup develop`` with setuptools installed. """ @@ -36,57 +37,59 @@ class TestManifestParser(unittest.TestCa # Show how you select subsets of tests: mozmill_restart_example = os.path.join(here, 'mozmill-restart-example.ini') parser.read(mozmill_restart_example) restart_tests = parser.get(type='restart') self.assertTrue(len(restart_tests) < len(parser.tests)) self.assertEqual(len(restart_tests), len(parser.get(manifest=mozmill_restart_example))) self.assertFalse([test for test in restart_tests - if test['manifest'] != os.path.join(here, 'mozmill-restart-example.ini')]) + if test['manifest'] != os.path.join(here, + 'mozmill-restart-example.ini')]) self.assertEqual(parser.get('name', tags=['foo']), ['restartTests/testExtensionInstallUninstall/test2.js', 'restartTests/testExtensionInstallUninstall/test1.js']) self.assertEqual(parser.get('name', foo='bar'), ['restartTests/testExtensionInstallUninstall/test2.js']) def test_include(self): """Illustrate how include works""" include_example = os.path.join(here, 'include-example.ini') parser = ManifestParser(manifests=(include_example,)) # All of the tests should be included, in order: self.assertEqual(parser.get('name'), ['crash-handling', 'fleem', 'flowers']) - self.assertEqual([(test['name'], os.path.basename(test['manifest'])) for test in parser.tests], - [('crash-handling', 'bar.ini'), ('fleem', 'include-example.ini'), ('flowers', 'foo.ini')]) + self.assertEqual([(test['name'], os.path.basename(test['manifest'])) + for test in parser.tests], + [('crash-handling', 'bar.ini'), + ('fleem', 'include-example.ini'), + ('flowers', 'foo.ini')]) # The including manifest is always reported as a part of the generated test object. self.assertTrue(all([t['ancestor-manifest'] == include_example for t in parser.tests if t['name'] != 'fleem'])) - # The manifests should be there too: self.assertEqual(len(parser.manifests()), 3) # We already have the root directory: self.assertEqual(here, parser.rootdir) - # DEFAULT values should persist across includes, unless they're # overwritten. In this example, include-example.ini sets foo=bar, but # it's overridden to fleem in bar.ini self.assertEqual(parser.get('name', foo='bar'), ['fleem', 'flowers']) self.assertEqual(parser.get('name', foo='fleem'), ['crash-handling']) # Passing parameters in the include section allows defining variables in - #the submodule scope: + # the submodule scope: self.assertEqual(parser.get('name', tags=['red']), ['flowers']) # However, this should be overridable from the DEFAULT section in the # included file and that overridable via the key directly connected to # the test: self.assertEqual(parser.get(name='flowers')[0]['blue'], 'ocean') @@ -102,40 +105,53 @@ class TestManifestParser(unittest.TestCa ['crash-handling', 'fleem']) # All of the included tests actually exist: self.assertEqual([i['name'] for i in parser.missing()], []) # Write the output to a manifest: buffer = StringIO() parser.write(fp=buffer, global_kwargs={'foo': 'bar'}) + expected_output = """[DEFAULT] +foo = bar + +[fleem] +subsuite = + +[include/flowers] +blue = ocean +red = roses +subsuite = +yellow = submarine""" # noqa + self.assertEqual(buffer.getvalue().strip(), - '[DEFAULT]\nfoo = bar\n\n[fleem]\nsubsuite = \n\n[include/flowers]\nblue = ocean\nred = roses\nsubsuite = \nyellow = submarine') + expected_output) def test_invalid_path(self): """ Test invalid path should not throw when not strict """ manifest = os.path.join(here, 'include-invalid.ini') - parser = ManifestParser(manifests=(manifest,), strict=False) + ManifestParser(manifests=(manifest,), strict=False) def test_parent_inheritance(self): """ Test parent manifest variable inheritance Specifically tests that inherited variables from parent includes properly propagate downstream """ parent_example = os.path.join(here, 'parent', 'level_1', 'level_2', 'level_3', 'level_3.ini') parser = ManifestParser(manifests=(parent_example,)) # Parent manifest test should not be included self.assertEqual(parser.get('name'), ['test_3']) - self.assertEqual([(test['name'], os.path.basename(test['manifest'])) for test in parser.tests], + self.assertEqual([(test['name'], os.path.basename(test['manifest'])) + for test in parser.tests], [('test_3', 'level_3.ini')]) # DEFAULT values should be the ones from level 1 self.assertEqual(parser.get('name', x='level_1'), ['test_3']) # Write the output to a manifest: buffer = StringIO() @@ -149,17 +165,18 @@ class TestManifestParser(unittest.TestCa """ parent_example = os.path.join(here, 'parent', 'level_1', 'level_2', 'level_3', 'level_3_default.ini') parser = ManifestParser(manifests=(parent_example,)) # Parent manifest test should not be included self.assertEqual(parser.get('name'), ['test_3']) - self.assertEqual([(test['name'], os.path.basename(test['manifest'])) for test in parser.tests], + self.assertEqual([(test['name'], os.path.basename(test['manifest'])) + for test in parser.tests], [('test_3', 'level_3_default.ini')]) # DEFAULT values should be the ones from level 3 self.assertEqual(parser.get('name', x='level_3'), ['test_3']) # Write the output to a manifest: buffer = StringIO() @@ -186,17 +203,17 @@ class TestManifestParser(unittest.TestCa Test server_root properly expands as an absolute path """ server_example = os.path.join(here, 'parent', 'level_1', 'level_2', 'level_3', 'level_3_server-root.ini') parser = ManifestParser(manifests=(server_example,)) # A regular variable will inherit its value directly self.assertEqual(parser.get('name', **{'other-root': '../root'}), - ['test_3']) + ['test_3']) # server-root will expand its value as an absolute path # we will not find anything for the original value self.assertEqual(parser.get('name', **{'server-root': '../root'}), []) # check that the path has expanded self.assertEqual(parser.get('server-root')[0], os.path.join(here, 'parent', 'root'))
--- a/testing/mozbase/manifestparser/tests/test_read_ini.py +++ b/testing/mozbase/manifestparser/tests/test_read_ini.py @@ -10,16 +10,17 @@ is the default: http://docs.python.org/2/library/configparser.html """ import unittest from manifestparser import read_ini from ConfigParser import ConfigParser from StringIO import StringIO + class IniParserTest(unittest.TestCase): def test_inline_comments(self): """ We have no inline comments; so we're testing to ensure we don't: https://bugzilla.mozilla.org/show_bug.cgi?id=855288 """
--- a/testing/mozbase/manifestparser/tests/test_testmanifest.py +++ b/testing/mozbase/manifestparser/tests/test_testmanifest.py @@ -13,19 +13,21 @@ here = os.path.dirname(os.path.abspath(_ class TestTestManifest(unittest.TestCase): """Test the Test Manifest""" def test_testmanifest(self): # Test filtering based on platform: filter_example = os.path.join(here, 'filter-example.ini') manifest = TestManifest(manifests=(filter_example,), strict=False) - self.assertEqual([i['name'] for i in manifest.active_tests(os='win', disabled=False, exists=False)], + self.assertEqual([i['name'] for i in manifest.active_tests(os='win', disabled=False, + exists=False)], ['windowstest', 'fleem']) - self.assertEqual([i['name'] for i in manifest.active_tests(os='linux', disabled=False, exists=False)], + self.assertEqual([i['name'] for i in manifest.active_tests(os='linux', disabled=False, + exists=False)], ['fleem', 'linuxtest']) # Look for existing tests. There is only one: self.assertEqual([i['name'] for i in manifest.active_tests()], ['fleem']) # You should be able to expect failures: last = manifest.active_tests(exists=False, toolkit='gtk2')[-1]
--- a/testing/mozbase/mozcrash/mozcrash/__init__.py +++ b/testing/mozbase/mozcrash/mozcrash/__init__.py @@ -1,8 +1,10 @@ +# flake8: noqa # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. """ -mozcrash is a library for getting a stack trace out of processes that have crashed and left behind a minidump file using the Google Breakpad library. +mozcrash is a library for getting a stack trace out of processes that have crashed +and left behind a minidump file using the Google Breakpad library. """ from mozcrash import *
--- a/testing/mozbase/mozcrash/mozcrash/mozcrash.py +++ b/testing/mozbase/mozcrash/mozcrash/mozcrash.py @@ -1,36 +1,36 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -__all__ = [ - 'check_for_crashes', - 'check_for_java_exception', - 'kill_and_get_minidump', - 'log_crashes', - 'cleanup_pending_crash_reports', -] - import glob import os import re import shutil import signal import subprocess import sys import tempfile import urllib2 import zipfile from collections import namedtuple import mozfile import mozinfo import mozlog +__all__ = [ + 'check_for_crashes', + 'check_for_java_exception', + 'kill_and_get_minidump', + 'log_crashes', + 'cleanup_pending_crash_reports', +] + StackInfo = namedtuple("StackInfo", ["minidump_path", "signature", "stackwalk_stdout", "stackwalk_stderr", "stackwalk_retcode", "stackwalk_errors", @@ -188,17 +188,18 @@ class CrashInfo(object): def dump_files(self): """List of tuple (path_to_dump_file, path_to_extra_file) for each dump file in self.dump_directory. The extra files may not exist.""" if self._dump_files is None: self._dump_files = [(path, os.path.splitext(path)[0] + '.extra') for path in glob.glob(os.path.join(self.dump_directory, '*.dmp'))] max_dumps = 10 if len(self._dump_files) > max_dumps: - self.logger.warning("Found %d dump files -- limited to %d!" % (len(self._dump_files), max_dumps)) + self.logger.warning("Found %d dump files -- limited to %d!" % + (len(self._dump_files), max_dumps)) del self._dump_files[max_dumps:] return self._dump_files @property def has_dumps(self): """Boolean indicating whether any crash dump files were found in the current directory""" @@ -233,17 +234,17 @@ class CrashInfo(object): errors = [] signature = None include_stderr = False out = None err = None retcode = None if (self.symbols_path and self.stackwalk_binary and os.path.exists(self.stackwalk_binary) and - os.access(self.stackwalk_binary, os.X_OK)): + os.access(self.stackwalk_binary, os.X_OK)): command = [ self.stackwalk_binary, path, self.symbols_path ] self.logger.info('Copy/paste: ' + ' '.join(command)) # run minidump_stackwalk @@ -257,22 +258,23 @@ class CrashInfo(object): if len(out) > 3: # minidump_stackwalk is chatty, # so ignore stderr when it succeeds. # The top frame of the crash is always the line after "Thread N (crashed)" # Examples: # 0 libc.so + 0xa888 # 0 libnss3.so!nssCertificate_Destroy [certificate.c : 102 + 0x0] - # 0 mozjs.dll!js::GlobalObject::getDebuggers() [GlobalObject.cpp:89df18f9b6da : 580 + 0x0] - # 0 libxul.so!void js::gc::MarkInternal<JSObject>(JSTracer*, JSObject**) [Marking.cpp : 92 + 0x28] + # 0 mozjs.dll!js::GlobalObject::getDebuggers() [GlobalObject.cpp:89df18f9b6da : 580 + 0x0] # noqa + # 0 libxul.so!void js::gc::MarkInternal<JSObject>(JSTracer*, JSObject**) + # [Marking.cpp : 92 + 0x28] lines = out.splitlines() for i, line in enumerate(lines): if "(crashed)" in line: - match = re.search(r"^ 0 (?:.*!)?(?:void )?([^\[]+)", lines[i+1]) + match = re.search(r"^ 0 (?:.*!)?(?:void )?([^\[]+)", lines[i + 1]) if match: signature = "@ %s" % match.group(1).strip() break else: include_stderr = True else: if not self.symbols_path: @@ -320,50 +322,52 @@ class CrashInfo(object): def check_for_java_exception(logcat, quiet=False): """ Print a summary of a fatal Java exception, if present in the provided logcat output. Example: - PROCESS-CRASH | java-exception | java.lang.NullPointerException at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833) + PROCESS-CRASH | java-exception | java.lang.NullPointerException at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833) # noqa `logcat` should be a list of strings. If `quiet` is set, no PROCESS-CRASH message will be printed to stdout if a crash is detected. Returns True if a fatal Java exception was found, False otherwise. """ found_exception = False for i, line in enumerate(logcat): # Logs will be of form: # - # 01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> REPORTING UNCAUGHT EXCEPTION FROM THREAD 9 ("GeckoBackgroundThread") + # 01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> REPORTING UNCAUGHT EXCEPTION FROM THREAD 9 ("GeckoBackgroundThread") # noqa # 01-30 20:15:41.937 E/GeckoAppShell( 1703): java.lang.NullPointerException - # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833) - # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at android.os.Handler.handleCallback(Handler.java:587) + # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833) # noqa + # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at android.os.Handler.handleCallback(Handler.java:587) # noqa if "REPORTING UNCAUGHT EXCEPTION" in line or "FATAL EXCEPTION" in line: # Strip away the date, time, logcat tag and pid from the next two lines and # concatenate the remainder to form a concise summary of the exception. found_exception = True if len(logcat) >= i + 3: logre = re.compile(r".*\): \t?(.*)") - m = logre.search(logcat[i+1]) + m = logre.search(logcat[i + 1]) if m and m.group(1): exception_type = m.group(1) - m = logre.search(logcat[i+2]) + m = logre.search(logcat[i + 2]) if m and m.group(1): exception_location = m.group(1) if not quiet: - print "PROCESS-CRASH | java-exception | %s %s" % (exception_type, exception_location) + print "PROCESS-CRASH | java-exception | %s %s" % (exception_type, + exception_location) else: - print "Automation Error: java exception in logcat at line %d of %d: %s" % (i, len(logcat), line) + print "Automation Error: java exception in logcat at line " \ + "%d of %d: %s" % (i, len(logcat), line) break return found_exception if mozinfo.isWin: import ctypes import uuid @@ -385,17 +389,17 @@ if mozinfo.isWin: CREATE_ALWAYS = 2 FILE_ATTRIBUTE_NORMAL = 0x80 INVALID_HANDLE_VALUE = -1 file_name = os.path.join(dump_directory, str(uuid.uuid4()) + ".dmp") if (mozinfo.info['bits'] != ctypes.sizeof(ctypes.c_voidp) * 8 and - utility_path): + utility_path): # We're not going to be able to write a minidump with ctypes if our # python process was compiled for a different architecture than # firefox, so we invoke the minidumpwriter utility program. log = get_logger() minidumpwriter = os.path.normpath(os.path.join(utility_path, "minidumpwriter.exe")) log.info("Using %s to write a dump to %s for [%d]" % @@ -460,16 +464,17 @@ else: def kill_pid(pid): """ Terminate a process with extreme prejudice. :param pid: PID of the process to terminate. """ os.kill(pid, signal.SIGKILL) + def kill_and_get_minidump(pid, dump_directory, utility_path=None): """ Attempt to kill a process and leave behind a minidump describing its execution state. :param pid: The PID of the process to kill. :param dump_directory: The directory where a minidump should be written on Windows, where the dump will be written from outside the process. @@ -487,16 +492,17 @@ def kill_and_get_minidump(pid, dump_dire if mozinfo.isWin: write_minidump(pid, dump_directory, utility_path) elif mozinfo.isLinux or mozinfo.isMac: os.kill(pid, signal.SIGABRT) needs_killing = False if needs_killing: kill_pid(pid) + def cleanup_pending_crash_reports(): """ Delete any pending crash reports. The presence of pending crash reports may be reported by the browser, affecting test results; it is best to ensure that these are removed before starting any browser tests.
--- a/testing/mozbase/mozcrash/setup.py +++ b/testing/mozbase/mozcrash/setup.py @@ -8,19 +8,20 @@ PACKAGE_NAME = 'mozcrash' PACKAGE_VERSION = '0.17' # dependencies deps = ['mozfile >= 1.0', 'mozlog >= 3.0'] setup(name=PACKAGE_NAME, version=PACKAGE_VERSION, - description="Library for printing stack traces from minidumps left behind by crashed processes", + description="Library for printing stack traces from minidumps " + "left behind by crashed processes", long_description="see http://mozbase.readthedocs.org/", - classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla', author='Mozilla Automation and Tools team', author_email='tools@lists.mozilla.org', url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', license='MPL', packages=['mozcrash'], include_package_data=True, zip_safe=False,
--- a/testing/mozbase/mozcrash/tests/test.py +++ b/testing/mozbase/mozcrash/tests/test.py @@ -1,41 +1,52 @@ #!/usr/bin/env python # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -import os, unittest, subprocess, tempfile, shutil, urlparse, zipfile, StringIO +import os +import unittest +import subprocess +import tempfile +import shutil +import urlparse +import zipfile +import StringIO import mozcrash import mozhttpd import mozlog.unstructured as mozlog # Make logs go away log = mozlog.getLogger("mozcrash", handler=mozlog.FileHandler(os.devnull)) + def popen_factory(stdouts): """ Generate a class that can mock subprocess.Popen. |stdouts| is an iterable that should return an iterable for the stdout of each process in turn. """ class mock_popen(object): + def __init__(self, args, *args_rest, **kwargs): self.stdout = stdouts.next() self.returncode = 0 def wait(self): return 0 def communicate(self): return (self.stdout.next(), "") return mock_popen + class TestCrash(unittest.TestCase): + def setUp(self): self.tempdir = tempfile.mkdtemp() # a fake file to use as a stackwalk binary self.stackwalk = os.path.join(self.tempdir, "stackwalk") open(self.stackwalk, "w").write("fake binary") self._subprocess_popen = subprocess.Popen subprocess.Popen = popen_factory(self.next_mock_stdout()) self.stdouts = [] @@ -168,61 +179,72 @@ class TestCrash(unittest.TestCase): self.stdouts.append(["this is some output"]) def make_zipfile(): data = StringIO.StringIO() z = zipfile.ZipFile(data, 'w') z.writestr("symbols.txt", "abc/xyz") z.close() return data.getvalue() + def get_symbols(req): headers = {} return (200, headers, make_zipfile()) httpd = mozhttpd.MozHttpd(port=0, - urlhandlers=[{'method':'GET', 'path':'/symbols', 'function':get_symbols}]) + urlhandlers=[{'method': 'GET', + 'path': '/symbols', + 'function': get_symbols}]) httpd.start() symbol_url = urlparse.urlunsplit(('http', '%s:%d' % httpd.httpd.server_address, - '/symbols','','')) + '/symbols', '', '')) self.assert_(mozcrash.check_for_crashes(self.tempdir, symbol_url, stackwalk_binary=self.stackwalk, quiet=True)) + class TestJavaException(unittest.TestCase): - def setUp(self): - self.test_log = ["01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> REPORTING UNCAUGHT EXCEPTION FROM THREAD 9 (\"GeckoBackgroundThread\")", - "01-30 20:15:41.937 E/GeckoAppShell( 1703): java.lang.NullPointerException", - "01-30 20:15:41.937 E/GeckoAppShell( 1703): at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833)", - "01-30 20:15:41.937 E/GeckoAppShell( 1703): at android.os.Handler.handleCallback(Handler.java:587)"] - def test_uncaught_exception(self): - """ - Test for an exception which should be caught - """ - self.assert_(mozcrash.check_for_java_exception(self.test_log, quiet=True)) + def setUp(self): + self.test_log = [ + "01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> " + "REPORTING UNCAUGHT EXCEPTION FROM THREAD 9 (\"GeckoBackgroundThread\")", + "01-30 20:15:41.937 E/GeckoAppShell( 1703): java.lang.NullPointerException", + "01-30 20:15:41.937 E/GeckoAppShell( 1703):" + " at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833)", + "01-30 20:15:41.937 E/GeckoAppShell( 1703):" + " at android.os.Handler.handleCallback(Handler.java:587)"] + + def test_uncaught_exception(self): + """ + Test for an exception which should be caught + """ + self.assert_(mozcrash.check_for_java_exception(self.test_log, quiet=True)) - def test_fatal_exception(self): - """ - Test for an exception which should be caught - """ - fatal_log = list(self.test_log) - fatal_log[0] = "01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> FATAL EXCEPTION FROM THREAD 9 (\"GeckoBackgroundThread\")" - self.assert_(mozcrash.check_for_java_exception(fatal_log, quiet=True)) + def test_fatal_exception(self): + """ + Test for an exception which should be caught + """ + fatal_log = list(self.test_log) + fatal_log[0] = "01-30 20:15:41.937 E/GeckoAppShell( 1703):" \ + " >>> FATAL EXCEPTION FROM THREAD 9 (\"GeckoBackgroundThread\")" + self.assert_(mozcrash.check_for_java_exception(fatal_log, quiet=True)) - def test_truncated_exception(self): - """ - Test for an exception which should be caught which - was truncated - """ - truncated_log = list(self.test_log) - truncated_log[0], truncated_log[1] = truncated_log[1], truncated_log[0] - self.assert_(mozcrash.check_for_java_exception(truncated_log, quiet=True)) + def test_truncated_exception(self): + """ + Test for an exception which should be caught which + was truncated + """ + truncated_log = list(self.test_log) + truncated_log[0], truncated_log[1] = truncated_log[1], truncated_log[0] + self.assert_(mozcrash.check_for_java_exception(truncated_log, quiet=True)) - def test_unchecked_exception(self): - """ - Test for an exception which should not be caught - """ - passable_log = list(self.test_log) - passable_log[0] = "01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> NOT-SO-BAD EXCEPTION FROM THREAD 9 (\"GeckoBackgroundThread\")" - self.assert_(not mozcrash.check_for_java_exception(passable_log, quiet=True)) + def test_unchecked_exception(self): + """ + Test for an exception which should not be caught + """ + passable_log = list(self.test_log) + passable_log[0] = "01-30 20:15:41.937 E/GeckoAppShell( 1703):" \ + " >>> NOT-SO-BAD EXCEPTION FROM THREAD 9 (\"GeckoBackgroundThread\")" + self.assert_(not mozcrash.check_for_java_exception(passable_log, quiet=True)) if __name__ == '__main__': unittest.main()
--- a/testing/mozbase/mozdebug/mozdebug/__init__.py +++ b/testing/mozbase/mozdebug/mozdebug/__init__.py @@ -1,8 +1,9 @@ +# flake8: noqa # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. """ This module contains a set of function to gather information about the debugging capabilities of the platform. It allows to look for a specific debugger or to query the system for a compatible/default debugger.
--- a/testing/mozbase/mozdebug/mozdebug/mozdebug.py +++ b/testing/mozbase/mozdebug/mozdebug/mozdebug.py @@ -56,36 +56,38 @@ To add support for a new debugger, simpl # Windows Development Kit super-debugger. 'windbg.exe': { 'interactive': True, }, } # Maps each OS platform to the preferred debugger programs found in _DEBUGGER_INFO. _DEBUGGER_PRIORITIES = { - 'win': ['devenv.exe', 'wdexpress.exe'], - 'linux': ['gdb', 'cgdb', 'lldb'], - 'mac': ['lldb', 'gdb'], - 'android': ['gdb'], - 'unknown': ['gdb'] + 'win': ['devenv.exe', 'wdexpress.exe'], + 'linux': ['gdb', 'cgdb', 'lldb'], + 'mac': ['lldb', 'gdb'], + 'android': ['gdb'], + 'unknown': ['gdb'] } + def _windbg_installation_paths(): programFilesSuffixes = ['', ' (x86)'] programFiles = "C:/Program Files" # Try the most recent versions first. windowsKitsVersions = ['10', '8.1', '8'] for suffix in programFilesSuffixes: windowsKitsPrefix = os.path.join(programFiles + suffix, 'Windows Kits') for version in windowsKitsVersions: yield os.path.join(windowsKitsPrefix, version, 'Debuggers', 'x86', 'windbg.exe') + def get_debugger_path(debugger): ''' Get the full path of the debugger. :param debugger: The name of the debugger. ''' if mozinfo.os == 'mac' and debugger == 'lldb': @@ -100,17 +102,18 @@ def get_debugger_path(debugger): if path: return path except: # Just default to find_executable instead. pass return find_executable(debugger) -def get_debugger_info(debugger, debuggerArgs = None, debuggerInteractive = False): + +def get_debugger_info(debugger, debuggerArgs=None, debuggerInteractive=False): ''' Get the information about the requested debugger. Returns a dictionary containing the |path| of the debugger executable, if it will run in |interactive| mode, its arguments and whether it needs to escape arguments it passes to the debugged program (|requiresEscapedArgs|). If the debugger cannot be found in the system, returns |None|. @@ -122,17 +125,17 @@ def get_debugger_info(debugger, debugger ''' debuggerPath = None if debugger: # Append '.exe' to the debugger on Windows if it's not present, # so things like '--debugger=devenv' work. if (os.name == 'nt' - and not debugger.lower().endswith('.exe')): + and not debugger.lower().endswith('.exe')): debugger += '.exe' debuggerPath = get_debugger_path(debugger) if not debuggerPath: # windbg is not installed with the standard set of tools, and it's # entirely possible that the user hasn't added the install location to # PATH, so we have to be a little more clever than normal to locate it. @@ -181,19 +184,22 @@ def get_debugger_info(debugger, debugger debugger_interactive, debugger_arguments, get_debugger_info('requiresEscapedArgs', False) ) return d # Defines the search policies to use in get_default_debugger_name. + + class DebuggerSearch: - OnlyFirst = 1 - KeepLooking = 2 + OnlyFirst = 1 + KeepLooking = 2 + def get_default_debugger_name(search=DebuggerSearch.OnlyFirst): ''' Get the debugger name for the default debugger on current platform. :param search: If specified, stops looking for the debugger if the default one is not found (|DebuggerSearch.OnlyFirst|) or keeps looking for other compatible debuggers (|DebuggerSearch.KeepLooking|). @@ -254,28 +260,32 @@ def get_default_debugger_name(search=Deb # TODO: pass in the path to the Valgrind to be used (--valgrind=), and # check what flags it accepts. Possible args that might be beneficial: # # --num-transtab-sectors=24 [reduces re-jitting overheads in long runs] # --px-default=allregs-at-mem-access # --px-file-backed=unwindregs-at-mem-access # [these reduce PX overheads as described above] # + + def get_default_valgrind_args(): return (['--fair-sched=yes', '--smc-check=all-non-file', '--vex-iropt-register-updates=allregs-at-mem-access', '--trace-children=yes', '--child-silent-after-fork=yes', ('--trace-children-skip=' + '/usr/bin/hg,/bin/rm,*/bin/certutil,*/bin/pk12util,' + '*/bin/ssltunnel,*/bin/uname,*/bin/which,*/bin/ps,' + '*/bin/grep,*/bin/java'), - ] + ] + get_default_valgrind_tool_specific_args()) # The default tool is Memcheck. Feeding these arguments to a different # Valgrind tool will cause it to fail at startup, so don't do that! + + def get_default_valgrind_tool_specific_args(): return ['--partial-loads-ok=yes', '--leak-check=full', '--show-possibly-lost=no', - ] + ]
--- a/testing/mozbase/mozdebug/setup.py +++ b/testing/mozbase/mozdebug/setup.py @@ -3,25 +3,25 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. from setuptools import setup PACKAGE_VERSION = '0.1' setup(name='mozdebug', version=PACKAGE_VERSION, - description="Utilities for running applications under native code debuggers intended for use in Mozilla testing", + description="Utilities for running applications under native code debuggers " + "intended for use in Mozilla testing", long_description="see http://mozbase.readthedocs.org/", - classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla', author='Mozilla Automation and Testing Team', author_email='tools@lists.mozilla.org', url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', license='MPL', packages=['mozdebug'], include_package_data=True, zip_safe=False, install_requires=['mozinfo'], entry_points=""" # -*- Entry points: -*- """, ) -
--- a/testing/mozbase/mozdevice/adb_tests/test_device_running_adb_as_root.py +++ b/testing/mozbase/mozdevice/adb_tests/test_device_running_adb_as_root.py @@ -5,17 +5,19 @@ Running this test case requires various reboots which makes it a very slow test case to run. """ import unittest import sys from mozdevice import DeviceManagerADB + class TestFileOperations(unittest.TestCase): + def setUp(self): dm = DeviceManagerADB() dm.reboot(wait=True) def test_run_adb_as_root_parameter(self): dm = DeviceManagerADB() self.assertTrue(dm.processInfo("adbd")[2] != "root") dm = DeviceManagerADB(runAdbAsRoot=True) @@ -29,18 +31,18 @@ class TestFileOperations(unittest.TestCa def tearDown(self): dm = DeviceManagerADB() dm.reboot() if __name__ == "__main__": dm = DeviceManagerADB() if not dm.devices(): - print "There are no connected adb devices" - sys.exit(1) + print "There are no connected adb devices" + sys.exit(1) else: - if not (int(dm._runCmd(["shell", "getprop", "ro.secure"]).output[0]) and \ + if not (int(dm._runCmd(["shell", "getprop", "ro.secure"]).output[0]) and int(dm._runCmd(["shell", "getprop", "ro.debuggable"]).output[0])): print "This test case is meant for devices with devices that start " \ - "adbd as non-root and allows for adbd to be restarted as root." + "adbd as non-root and allows for adbd to be restarted as root." sys.exit(1) unittest.main()
--- a/testing/mozbase/mozdevice/adb_tests/test_devicemanagerADB.py +++ b/testing/mozbase/mozdevice/adb_tests/test_devicemanagerADB.py @@ -39,21 +39,23 @@ import re import socket import sys import tempfile import unittest from StringIO import StringIO from mozdevice import DeviceManagerADB, DMError + def find_mount_permissions(dm, mount_path): for mount_point in dm._runCmd(["shell", "mount"]).output: if mount_point.find(mount_path) > 0: return re.search('(ro|rw)(?=,)', mount_point).group(0) + class DeviceManagerADBTestCase(unittest.TestCase): tempLocalDir = "tempDir" tempLocalFile = os.path.join(tempLocalDir, "tempfile.txt") tempRemoteDir = None tempRemoteFile = None tempRemoteSystemFile = None def setUp(self): @@ -76,32 +78,33 @@ class DeviceManagerADBTestCase(unittest. self.dm = DeviceManagerADB() if not os.path.exists(self.tempLocalDir): os.mkdir(self.tempLocalDir) if not os.path.exists(self.tempLocalFile): # Create empty file open(self.tempLocalFile, 'w').close() self.tempRemoteDir = self.dm.getTempDir() self.tempRemoteFile = os.path.join(self.tempRemoteDir, - os.path.basename(self.tempLocalFile)) + os.path.basename(self.tempLocalFile)) self.tempRemoteSystemFile = \ os.path.join("/system", os.path.basename(self.tempLocalFile)) @classmethod def tearDownClass(self): os.remove(self.tempLocalFile) os.rmdir(self.tempLocalDir) if self.dm.dirExists(self.tempRemoteDir): # self.tempRemoteFile will get deleted with it self.dm.removeDir(self.tempRemoteDir) if self.dm.fileExists(self.tempRemoteSystemFile): self.dm.removeFile(self.tempRemoteSystemFile) class TestFileOperations(DeviceManagerADBTestCase): + def test_make_and_remove_directory(self): dir1 = os.path.join(self.tempRemoteDir, "dir1") self.assertFalse(self.dm.dirExists(dir1)) self.dm.mkDir(dir1) self.assertTrue(self.dm.dirExists(dir1)) self.dm.removeDir(dir1) self.assertFalse(self.dm.dirExists(dir1)) @@ -163,32 +166,33 @@ class TestFileOperations(DeviceManagerAD self.dm.removeFile(self.tempRemoteSystemFile) self.assertFalse(self.dm.fileExists(self.tempRemoteSystemFile)) self.dm.shell(['mount', '-r', '-o', 'remount', '/system'], out) out.close() self.assertTrue(find_mount_permissions(self.dm, "/system") == "ro") class TestOther(DeviceManagerADBTestCase): + def test_get_list_of_processes(self): self.assertEquals(type(self.dm.getProcessList()), list) def test_get_current_time(self): self.assertEquals(type(self.dm.getCurrentTime()), int) def test_get_info(self): self.assertEquals(type(self.dm.getInfo()), dict) def test_list_devices(self): self.assertEquals(len(list(self.dm.devices())), 1) def test_shell(self): out = StringIO() self.dm.shell(["echo", "$COMPANY", ";", "pwd"], out, - env={"COMPANY":"Mozilla"}, cwd="/", timeout=4, root=True) + env={"COMPANY": "Mozilla"}, cwd="/", timeout=4, root=True) output = str(out.getvalue()).rstrip().splitlines() out.close() self.assertEquals(output, ['Mozilla', '/']) def test_port_forwarding(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("", 0)) port = s.getsockname()[1] @@ -198,18 +202,18 @@ class TestOther(DeviceManagerADBTestCase def test_port_forwarding_error(self): self.assertRaises(DMError, self.dm.forward, "", "") if __name__ == '__main__': dm = DeviceManagerADB() if not dm.devices(): - print "There are no connected adb devices" - sys.exit(1) + print "There are no connected adb devices" + sys.exit(1) if find_mount_permissions(dm, "/system") == "rw": print "We've found out that /system is mounted as 'rw'. This is because the command " \ - "'adb remount' has been run before running this test case. Please reboot the device " \ - "and try again." + "'adb remount' has been run before running this test case. Please reboot the device " \ + "and try again." sys.exit(1) unittest.main()
--- a/testing/mozbase/mozdevice/mozdevice/__init__.py +++ b/testing/mozbase/mozdevice/mozdevice/__init__.py @@ -4,8 +4,12 @@ from adb import ADBError, ADBRootError, ADBTimeoutError, ADBProcess, ADBCommand, ADBHost, ADBDevice from adb_android import ADBAndroid from adb_b2g import ADBB2G from devicemanager import DeviceManager, DMError, ZeroconfListener from devicemanagerADB import DeviceManagerADB from devicemanagerSUT import DeviceManagerSUT from droid import DroidADB, DroidSUT, DroidConnectByHWID + +__all__ = ['ADBError', 'ADBRootError', 'ADBTimeoutError', 'ADBProcess', 'ADBCommand', 'ADBHost', + 'ADBDevice', 'ADBAndroid', 'ADBB2G', 'DeviceManager', 'DMError', 'ZeroconfListener', + 'DeviceManagerADB', 'DeviceManagerSUT', 'DroidADB', 'DroidSUT', 'DroidConnectByHWID']
--- a/testing/mozbase/mozdevice/mozdevice/adb.py +++ b/testing/mozbase/mozdevice/mozdevice/adb.py @@ -10,16 +10,17 @@ import tempfile import time import traceback from abc import ABCMeta, abstractmethod class ADBProcess(object): """ADBProcess encapsulates the data related to executing the adb process.""" + def __init__(self, args): #: command argument argument list. self.args = args #: Temporary file handle to be used for stdout. self.stdout_file = tempfile.TemporaryFile() #: Temporary file handle to be used for stderr. self.stderr_file = tempfile.TemporaryFile() #: boolean indicating if the command timed out. @@ -54,42 +55,47 @@ class ADBProcess(object): def __str__(self): return ('args: %s, exitcode: %s, stdout: %s, stderr: %s' % ( ' '.join(self.args), self.exitcode, self.stdout, self.stderr)) # ADBError, ADBRootError, and ADBTimeoutError are treated # differently in order that unhandled ADBRootErrors and # ADBTimeoutErrors can be handled distinctly from ADBErrors. + class ADBError(Exception): """ADBError is raised in situations where a command executed on a device either exited with a non-zero exitcode or when an unexpected error condition has occurred. Generally, ADBErrors can be handled and the device can continue to be used. """ pass + class ADBListDevicesError(ADBError): """ADBListDevicesError is raised when errors are found listing the devices, typically not any permissions. The devices information is stocked with the *devices* member. """ + def __init__(self, msg, devices): ADBError.__init__(self, msg) self.devices = devices + class ADBRootError(Exception): """ADBRootError is raised when a shell command is to be executed as root but the device does not support it. This error is fatal since there is no recovery possible by the script. You must either root your device or change your scripts to not require running as root. """ pass + class ADBTimeoutError(Exception): """ADBTimeoutError is raised when either a host command or shell command takes longer than the specified timeout to execute. The timeout value is set in the ADBCommand constructor and is 300 seconds by default. This error is typically fatal since the host is having problems communicating with the device. You may be able to recover by rebooting, but this is not guaranteed. @@ -229,20 +235,20 @@ class ADBCommand(object): adb_process = ADBProcess(args) if timeout is None: timeout = self._timeout start_time = time.time() adb_process.exitcode = adb_process.proc.poll() while ((time.time() - start_time) <= timeout and - adb_process.exitcode == None): + adb_process.exitcode is None): time.sleep(self._polling_interval) adb_process.exitcode = adb_process.proc.poll() - if adb_process.exitcode == None: + if adb_process.exitcode is None: adb_process.proc.kill() adb_process.timedout = True adb_process.exitcode = adb_process.proc.poll() adb_process.stdout_file.seek(0, os.SEEK_SET) adb_process.stderr_file.seek(0, os.SEEK_SET) return adb_process @@ -305,16 +311,17 @@ class ADBHost(ADBCommand): :: from mozdevice import ADBHost adbhost = ADBHost() adbhost.start_server() """ + def __init__(self, adb='adb', adb_host=None, adb_port=None, logger_name='adb', timeout=300, verbose=False): """Initializes the ADBHost object. @@ -453,17 +460,19 @@ class ADBHost(ADBCommand): is parsed and placed into an object as in [{'device_serial': 'b313b945', 'state': 'device', 'product': 'd2vzw', 'usb': '1-7', 'device': 'd2vzw', 'model': 'SCH_I535' }] """ # b313b945 device usb:1-7 product:d2vzw model:SCH_I535 device:d2vzw # from Android system/core/adb/transport.c statename() - re_device_info = re.compile(r'([^\s]+)\s+(offline|bootloader|device|host|recovery|sideload|no permissions|unauthorized|unknown)') + re_device_info = re.compile( + r"([^\s]+)\s+(offline|bootloader|device|host|recovery|sideload|" + "no permissions|unauthorized|unknown)") devices = [] lines = self.command_output(["devices", "-l"], timeout=timeout).split('\n') for line in lines: if line == 'List of devices attached ': continue match = re_device_info.match(line) if match: device = { @@ -660,17 +669,16 @@ class ADBDevice(ADBCommand): self.command_output( ["root"], timeout=timeout).find("cannot run as root") == -1): self._have_root_shell = True self._logger.info("adbd restarted as root") except ADBError: self._logger.debug("Check for root adbd failed") - @staticmethod def _escape_command_line(cmd): """Utility function to return escaped and quoted version of command line. """ quoted_cmd = [] for arg in cmd: @@ -1034,20 +1042,20 @@ class ADBDevice(ADBCommand): args.extend(["wait-for-device", "shell", cmd]) adb_process = ADBProcess(args) if timeout is None: timeout = self._timeout start_time = time.time() exitcode = adb_process.proc.poll() - while ((time.time() - start_time) <= timeout) and exitcode == None: + while ((time.time() - start_time) <= timeout) and exitcode is None: time.sleep(self._polling_interval) exitcode = adb_process.proc.poll() - if exitcode == None: + if exitcode is None: adb_process.proc.kill() adb_process.timedout = True adb_process.exitcode = adb_process.proc.poll() elif exitcode == 0: adb_process.exitcode = self._get_exitcode(adb_process.stdout_file) else: adb_process.exitcode = exitcode @@ -1125,22 +1133,22 @@ class ADBDevice(ADBCommand): output = adb_process.stdout_file.read().rstrip() if self._verbose: self._logger.debug('shell_output: %s, ' 'timeout: %s, ' 'root: %s, ' 'timedout: %s, ' 'exitcode: %s, ' 'output: %s' % - (' '.join(adb_process.args), - timeout, + (' '.join(adb_process.args), + timeout, root, - adb_process.timedout, - adb_process.exitcode, - output)) + adb_process.timedout, + adb_process.exitcode, + output)) return output finally: if adb_process and isinstance(adb_process.stdout_file, file): adb_process.stdout_file.close() adb_process.stderr_file.close() # Informational methods @@ -1693,17 +1701,17 @@ class ADBDevice(ADBCommand): raise pid_set = set(pid_list) current_pid_set = set([str(proc[0]) for proc in self.get_process_list(timeout=timeout)]) pid_list = list(pid_set.intersection(current_pid_set)) if not pid_list: break self._logger.debug("Attempt %d of %d to kill processes %s failed" % - (attempt+1, attempts, pid_list)) + (attempt + 1, attempts, pid_list)) time.sleep(wait) if pid_list: raise ADBError('kill: processes %s not killed' % pid_list) def pkill(self, appname, sig=None, attempts=3, wait=5, timeout=None, root=False): """Kills a processes on the device matching a name.
--- a/testing/mozbase/mozdevice/mozdevice/adb_android.py +++ b/testing/mozbase/mozdevice/mozdevice/adb_android.py @@ -148,17 +148,17 @@ class ADBAndroid(ADBDevice): if match: parameter = match.group(1) value = match.group(2) if parameter == 'level': level = float(value) elif parameter == 'scale': scale = float(value) if parameter is not None and scale is not None: - percentage = 100.0*level/scale + percentage = 100.0 * level / scale break return percentage # System control methods def is_device_ready(self, timeout=None): """Checks if a device is ready for testing. @@ -214,17 +214,17 @@ class ADBAndroid(ADBDevice): success = False break except ADBError as e: success = False failure = e.message if not success: self._logger.debug('Attempt %s of %s device not ready: %s' % ( - attempt+1, self._device_ready_retry_attempts, + attempt + 1, self._device_ready_retry_attempts, failure)) time.sleep(self._device_ready_retry_wait) return success def power_on(self, timeout=None): """Sets the device's power stayon value. @@ -288,18 +288,18 @@ class ADBAndroid(ADBDevice): data = self.shell_output("pm list package %s" % app_name, timeout=timeout) if pm_error_string in data: raise ADBError(pm_error_string) if app_name not in data: return False return True def launch_application(self, app_name, activity_name, intent, url=None, - extras=None, wait=True, fail_if_running=True, - timeout=None): + extras=None, wait=True, fail_if_running=True, + timeout=None): """Launches an Android application :param str app_name: Name of application (e.g. `com.android.chrome`) :param str activity_name: Name of activity to launch (e.g. `.Main`) :param str intent: Intent to launch application with :param url: URL to open :type url: str or None :param extras: Extra arguments for application. @@ -321,17 +321,17 @@ class ADBAndroid(ADBDevice): # If fail_if_running is True, we throw an exception here. Only one # instance of an application can be running at once on Android, # starting a new instance may not be what we want depending on what # we want to do if fail_if_running and self.process_exist(app_name, timeout=timeout): raise ADBError("Only one instance of an application may be running " "at once") - acmd = [ "am", "start" ] + \ + acmd = ["am", "start"] + \ ["-W" if wait else '', "-n", "%s/%s" % (app_name, activity_name)] if intent: acmd.extend(["-a", intent]) if extras: for (key, val) in extras.iteritems(): if type(val) is int: @@ -344,18 +344,18 @@ class ADBAndroid(ADBDevice): if url: acmd.extend(["-d", url]) cmd = self._escape_command_line(acmd) self.shell_output(cmd, timeout=timeout) def launch_fennec(self, app_name, intent="android.intent.action.VIEW", - moz_env=None, extra_args=None, url=None, wait=True, - fail_if_running=True, timeout=None): + moz_env=None, extra_args=None, url=None, wait=True, + fail_if_running=True, timeout=None): """Convenience method to launch Fennec on Android with various debugging arguments :param str app_name: Name of fennec application (e.g. `org.mozilla.fennec`) :param str intent: Intent to launch application. :param moz_env: Mozilla specific environment to pass into application. @@ -386,19 +386,20 @@ class ADBAndroid(ADBDevice): for (env_count, (env_key, env_val)) in enumerate(moz_env.iteritems()): extras["env" + str(env_count)] = env_key + "=" + env_val # Additional command line arguments that fennec will read and use (e.g. # with a custom profile) if extra_args: extras['args'] = " ".join(extra_args) - self.launch_application(app_name, "org.mozilla.gecko.BrowserApp", intent, url=url, extras=extras, - wait=wait, fail_if_running=fail_if_running, - timeout=timeout) + self.launch_application(app_name, "org.mozilla.gecko.BrowserApp", intent, url=url, + extras=extras, + wait=wait, fail_if_running=fail_if_running, + timeout=timeout) def stop_application(self, app_name, timeout=None, root=False): """Stops the specified application For Android 3.0+, we use the "am force-stop" to do this, which is reliable and does not require root. For earlier versions of Android, we simply try to manually kill the processes started by the app repeatedly until none is around any more. This is @@ -423,17 +424,17 @@ class ADBAndroid(ADBDevice): self.shell_output("am force-stop %s" % app_name, timeout=timeout, root=root) else: num_tries = 0 max_tries = 5 while self.process_exist(app_name, timeout=timeout): if num_tries > max_tries: raise ADBError("Couldn't successfully kill %s after %s " - "tries" % (app_name, max_tries)) + "tries" % (app_name, max_tries)) self.pkill(app_name, timeout=timeout, root=root) num_tries += 1 # sleep for a short duration to make sure there are no # additional processes in the process of being launched # (this is not 100% guaranteed to work since it is inherently # racey, but it's the best we can do) time.sleep(1)
--- a/testing/mozbase/mozdevice/mozdevice/devicemanager.py +++ b/testing/mozbase/mozdevice/mozdevice/devicemanager.py @@ -9,36 +9,40 @@ import os import posixpath import re import struct import StringIO import zlib from functools import wraps + class DMError(Exception): "generic devicemanager exception." - def __init__(self, msg= '', fatal = False): + def __init__(self, msg='', fatal=False): self.msg = msg self.fatal = fatal def __str__(self): return self.msg + def abstractmethod(method): line = method.func_code.co_firstlineno filename = method.func_code.co_filename + @wraps(method) def not_implemented(*args, **kwargs): raise NotImplementedError('Abstract method %s at File "%s", line %s ' - 'should be implemented by a concrete class' % - (repr(method), filename, line)) + 'should be implemented by a concrete class' % + (repr(method), filename, line)) return not_implemented + class DeviceManager(object): """ Represents a connection to a device. Once an implementation of this class is successfully instantiated, you may do things like list/copy files to the device, launch processes on the device, and install or remove applications from the device. Never instantiate this class directly! Instead, instantiate an @@ -47,17 +51,17 @@ class DeviceManager(object): _logcatNeedsRoot = True default_timeout = 300 short_timeout = 30 def __init__(self, logLevel=None, deviceRoot=None): try: self._logger = mozlog.get_default_logger(component="mozdevice") - if not self._logger: # no global structured logger, fall back to reg logging + if not self._logger: # no global structured logger, fall back to reg logging self._logger = mozlog.unstructured.getLogger("mozdevice") if logLevel is not None: self._logger.setLevel(logLevel) except AttributeError: # Structured logging doesn't work on Python 2.6 self._logger = None self._logLevel = logLevel self._remoteIsWin = None @@ -91,32 +95,33 @@ class DeviceManager(object): self._logger.warning("dm.debug is deprecated. Use logLevel.") levels = {logging.DEBUG: 5, logging.INFO: 3, logging.WARNING: 2, logging.ERROR: 1, logging.CRITICAL: 0} return levels[self.logLevel] @debug.setter def debug_setter(self, newDebug): self._logger.warning("dm.debug is deprecated. Use logLevel.") - newDebug = 5 if newDebug > 5 else newDebug # truncate >=5 to 5 + newDebug = 5 if newDebug > 5 else newDebug # truncate >=5 to 5 levels = {5: logging.DEBUG, 3: logging.INFO, 2: logging.WARNING, 1: logging.ERROR, 0: logging.CRITICAL} self.logLevel = levels[newDebug] @abstractmethod def getInfo(self, directive=None): """ Returns a dictionary of information strings about the device. :param directive: information you want to get. Options are: - `os` - name of the os - `id` - unique id of the device - `uptime` - uptime of the device - - `uptimemillis` - uptime of the device in milliseconds (NOT supported on all implementations) + - `uptimemillis` - uptime of the device in milliseconds + (NOT supported on all implementations) - `systime` - system time of the device - `screen` - screen resolution - `memory` - memory stats - `memtotal` - total memory available on the device, for example 927208 kB - `process` - list of running processes (same as ps) - `disk` - total, free, available bytes on disk - `power` - power status (charge, battery temp) - `temperature` - device temperature @@ -132,43 +137,45 @@ class DeviceManager(object): def getIP(self, interfaces=['eth0', 'wlan0']): """ Returns the IP of the device, or None if no connection exists. """ for interface in interfaces: match = re.match(r"%s: ip (\S+)" % interface, self.shellCheckOutput(['ifconfig', interface], - timeout=self.short_timeout)) + timeout=self.short_timeout)) if match: return match.group(1) def recordLogcat(self): """ Clears the logcat file making it easier to view specific events. """ - #TODO: spawn this off in a separate thread/process so we can collect all the logcat information + # TODO: spawn this off in a separate thread/process so we can collect all + # the logcat information - # Right now this is just clearing the logcat so we can only see what happens after this call. + # Right now this is just clearing the logcat so we can only see what + # happens after this call. self.shellCheckOutput(['/system/bin/logcat', '-c'], root=self._logcatNeedsRoot, timeout=self.short_timeout) def getLogcat(self, filterSpecs=["dalvikvm:I", "ConnectivityService:S", - "WifiMonitor:S", "WifiStateTracker:S", - "wpa_supplicant:S", "NetworkStateTracker:S"], + "WifiMonitor:S", "WifiStateTracker:S", + "wpa_supplicant:S", "NetworkStateTracker:S"], format="time", filterOutRegexps=[]): """ Returns the contents of the logcat file as a list of '\n' terminated strings """ cmdline = ["/system/bin/logcat", "-v", format, "-d"] + filterSpecs output = self.shellCheckOutput(cmdline, - root=self._logcatNeedsRoot, - timeout=self.short_timeout) + root=self._logcatNeedsRoot, + timeout=self.short_timeout) lines = output.replace('\r\n', '\n').splitlines(True) for regex in filterOutRegexps: lines = [line for line in lines if not re.search(regex, line)] return lines def saveScreenshot(self, filename): @@ -247,18 +254,18 @@ class DeviceManager(object): for root, dirs, files in os.walk(localDirname): parts = root.split(localDirname) for f in files: remoteRoot = remoteDirname + '/' + parts[1] remoteRoot = remoteRoot.replace('/', '/') if (parts[1] == ""): remoteRoot = remoteDirname remoteName = remoteRoot + '/' + f - if (self.validateFile(remoteName, os.path.join(root, f)) <> True): - return False + if (self.validateFile(remoteName, os.path.join(root, f)) is not True): + return False return True @abstractmethod def mkDir(self, remoteDirname): """ Creates a single directory on the device file system. """ @@ -272,17 +279,17 @@ class DeviceManager(object): filename = posixpath.normpath(filename) containing = posixpath.dirname(filename) if not self.dirExists(containing): parts = filename.split('/') name = "/" if not self.remoteIsWin else parts.pop(0) for part in parts[:-1]: if part != "": name = posixpath.join(name, part) - self.mkDir(name) # mkDir will check previous existence + self.mkDir(name) # mkDir will check previous existence @abstractmethod def dirExists(self, dirpath): """ Returns whether dirpath exists and is a directory on the device file system. """ @abstractmethod @@ -309,31 +316,31 @@ class DeviceManager(object): @abstractmethod def removeDir(self, remoteDirname): """ Does a recursive delete of directory on the device: rm -Rf remoteDirname. """ @abstractmethod def moveTree(self, source, destination): - """ - Does a move of the file or directory on the device. + """ + Does a move of the file or directory on the device. - :param source: Path to the original file or directory - :param destination: Path to the destination file or directory - """ + :param source: Path to the original file or directory + :param destination: Path to the destination file or directory + """ @abstractmethod def copyTree(self, source, destination): - """ - Does a copy of the file or directory on the device. + """ + Does a copy of the file or directory on the device. - :param source: Path to the original file or directory - :param destination: Path to the destination file or directory - """ + :param source: Path to the original file or directory + :param destination: Path to the destination file or directory + """ @abstractmethod def chmodDir(self, remoteDirname, mask="777"): """ Recursively changes file permissions in a directory. """ @property @@ -397,17 +404,19 @@ class DeviceManager(object): :param root: Specifies whether command requires root privileges :raises: DMError """ buf = StringIO.StringIO() retval = self.shell(cmd, buf, env=env, cwd=cwd, timeout=timeout, root=root) output = str(buf.getvalue()[0:-1]).rstrip() buf.close() if retval != 0: - raise DMError("Non-zero return code for command: %s (output: '%s', retval: '%s')" % (cmd, output, retval)) + raise DMError( + "Non-zero return code for command: %s " + "(output: '%s', retval: '%s')" % (cmd, output, retval)) return output @abstractmethod def getProcessList(self): """ Returns array of tuples representing running processes on the device. Format of tuples is (processId, processName, userId) @@ -419,22 +428,22 @@ class DeviceManager(object): Information on process is in tuple format: (pid, process path, user) If a process with the specified name does not exist this function will return None. """ if not isinstance(processName, basestring): raise TypeError("Process name %s is not a string" % processName) processInfo = None - #filter out extra spaces + # filter out extra spaces parts = filter(lambda x: x != '', processName.split(' ')) processName = ' '.join(parts) - #filter out the quoted env string if it exists - #ex: '"name=value;name2=value2;etc=..." process args' -> 'process args' + # filter out the quoted env string if it exists + # ex: '"name=value;name2=value2;etc=..." process args' -> 'process args' parts = processName.split('"') if (len(parts) > 2): processName = ' '.join(parts[2:]).strip() pieces = processName.split(' ') parts = pieces[0].split('/') app = parts[-1] @@ -480,17 +489,18 @@ class DeviceManager(object): """ @abstractmethod def installApp(self, appBundlePath, destPath=None): """ Installs an application onto the device. :param appBundlePath: path to the application bundle on the device - :param destPath: destination directory of where application should be installed to (optional) + :param destPath: destination directory of where application should be + installed to (optional) """ @abstractmethod def uninstallApp(self, appName, installPath=None): """ Uninstalls the named application from device and DOES NOT cause a reboot. :param appName: the name of the application (e.g org.mozilla.fennec) @@ -526,39 +536,43 @@ class DeviceManager(object): @staticmethod def _writePNG(buf, width, height): """ Method for writing a PNG from a buffer, used by getScreenshot on older devices, """ # Based on: http://code.activestate.com/recipes/577443-write-a-png-image-in-native-python/ width_byte_4 = width * 4 - raw_data = b"".join(b'\x00' + buf[span:span + width_byte_4] for span in range(0, (height - 1) * width * 4, width_byte_4)) + raw_data = b"".join(b'\x00' + buf[span:span + width_byte_4] + for span in range(0, (height - 1) * width * 4, width_byte_4)) + def png_pack(png_tag, data): chunk_head = png_tag + data - return struct.pack("!I", len(data)) + chunk_head + struct.pack("!I", 0xFFFFFFFF & zlib.crc32(chunk_head)) + return struct.pack("!I", len(data)) \ + + chunk_head \ + + struct.pack("!I", 0xFFFFFFFF & zlib.crc32(chunk_head)) return b"".join([ - b'\x89PNG\r\n\x1a\n', - png_pack(b'IHDR', struct.pack("!2I5B", width, height, 8, 6, 0, 0, 0)), - png_pack(b'IDAT', zlib.compress(raw_data, 9)), - png_pack(b'IEND', b'')]) + b'\x89PNG\r\n\x1a\n', + png_pack(b'IHDR', struct.pack("!2I5B", width, height, 8, 6, 0, 0, 0)), + png_pack(b'IDAT', zlib.compress(raw_data, 9)), + png_pack(b'IEND', b'')]) @abstractmethod def _getRemoteHash(self, filename): """ Return the md5 sum of a file on the device. """ @staticmethod def _getLocalHash(filename): """ Return the MD5 sum of a file on the host. """ f = open(filename, 'rb') - if (f == None): + if f is None: return None try: mdsum = hashlib.md5() except: return None while 1: @@ -577,67 +591,70 @@ class DeviceManager(object): Utility function to return escaped and quoted version of command line. """ quotedCmd = [] for arg in cmd: arg.replace('&', '\&') needsQuoting = False - for char in [ ' ', '(', ')', '"', '&' ]: + for char in [' ', '(', ')', '"', '&']: if arg.find(char) >= 0: needsQuoting = True break if needsQuoting: arg = '\'%s\'' % arg quotedCmd.append(arg) return " ".join(quotedCmd) + def _pop_last_line(file_obj): """ Utility function to get the last line from a file (shared between ADB and SUT device managers). Function also removes it from the file. Intended to strip off the return code from a shell command. """ bytes_from_end = 1 file_obj.seek(0, 2) length = file_obj.tell() + 1 while bytes_from_end < length: - file_obj.seek((-1)*bytes_from_end, 2) + file_obj.seek((-1) * bytes_from_end, 2) data = file_obj.read() - if bytes_from_end == length-1 and len(data) == 0: # no data, return None + if bytes_from_end == length - 1 and len(data) == 0: # no data, return None return None - if data[0] == '\n' or bytes_from_end == length-1: + if data[0] == '\n' or bytes_from_end == length - 1: # found the last line, which should have the return value if data[0] == '\n': data = data[1:] # truncate off the return code line file_obj.truncate(length - bytes_from_end) - file_obj.seek(0,2) + file_obj.seek(0, 2) file_obj.write('\0') return data bytes_from_end += 1 return None + class ZeroconfListener(object): + def __init__(self, hwid, evt): self.hwid = hwid self.evt = evt # Format is 'SUTAgent [hwid:015d2bc2825ff206] [ip:10_242_29_221]._sutagent._tcp.local.' def addService(self, zeroconf, type, name): - #print "Found _sutagent service broadcast:", name + # print "Found _sutagent service broadcast:", name if not name.startswith("SUTAgent"): return sutname = name.split('.')[0] m = re.search('\[hwid:([^\]]*)\]', sutname) if m is None: return
--- a/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py +++ b/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py @@ -126,50 +126,51 @@ class DeviceManagerADB(DeviceManager): # prepend cwd and env to command if necessary if cwd: cmdline = "cd %s; %s" % (cwd, cmdline) if env: envstr = '; '.join(map(lambda x: 'export %s=%s' % (x[0], x[1]), env.iteritems())) cmdline = envstr + "; " + cmdline # all output should be in stdout - args=[self._adbPath] + args = [self._adbPath] if self._serverHost is not None: args.extend(['-H', self._serverHost]) if self._serverPort is not None: args.extend(['-P', str(self._serverPort)]) if self._deviceSerial: args.extend(['-s', self._deviceSerial]) args.extend(["shell", cmdline]) def _timeout(): self._logger.error("Timeout exceeded for shell call '%s'" % ' '.join(args)) self._logger.debug("shell - command: %s" % ' '.join(args)) proc = ProcessHandler(args, processOutputLine=self._log, onTimeout=_timeout) if not timeout: - # We are asserting that all commands will complete in this time unless otherwise specified + # We are asserting that all commands will complete in this time unless + # otherwise specified timeout = self.default_timeout timeout = int(timeout) proc.run(timeout) proc.wait() output = proc.output if output: lastline = output[-1] if lastline: m = re.search('([0-9]+)', lastline) if m: return_code = m.group(1) for line in output: outputfile.write(line + '\n') outputfile.seek(-2, 2) - outputfile.truncate() # truncate off the return code + outputfile.truncate() # truncate off the return code return int(return_code) return None def forward(self, local, remote): """ Forward socket connections. @@ -191,25 +192,24 @@ class DeviceManagerADB(DeviceManager): cmd = ['forward'] if local is None: cmd.extend(['--remove-all']) else: cmd.extend(['--remove', local]) if not self._checkCmd(cmd, timeout=self.short_timeout) == 0: raise DMError("Failed to remove connection forwarding.") - def remount(self): "Remounts the /system partition on the device read-write." return self._checkCmd(['remount'], timeout=self.short_timeout) def devices(self): "Return a list of connected devices as (serial, status) tuples." proc = self._runCmd(['devices']) - proc.output.pop(0) # ignore first line of output + proc.output.pop(0) # ignore first line of output devices = [] for line in proc.output: result = re.match('(.*?)\t(.*)', line) if result: devices.append((result.group(1), result.group(2))) return devices def _connectRemoteADB(self): @@ -226,39 +226,40 @@ class DeviceManagerADB(DeviceManager): retryLimit = retryLimit or self.retryLimit if self.dirExists(destname): raise DMError("Attempted to push a file (%s) to a directory (%s)!" % (localname, destname)) if not os.access(localname, os.F_OK): raise DMError("File not found: %s" % localname) proc = self._runCmd(["push", os.path.realpath(localname), destname], - retryLimit=retryLimit) + retryLimit=retryLimit) if proc.returncode != 0: - raise DMError("Error pushing file %s -> %s; output: %s" % (localname, destname, proc.output)) + raise DMError("Error pushing file %s -> %s; output: %s" % + (localname, destname, proc.output)) def mkDir(self, name): result = self._runCmd(["shell", "mkdir", name], timeout=self.short_timeout).output if len(result) and 'read-only file system' in result[0].lower(): raise DMError("Error creating directory: read only file system") def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None): # adb "push" accepts a directory as an argument, but if the directory # contains symbolic links, the links are pushed, rather than the linked # files; we either zip/unzip or re-copy the directory into a temporary # one to get around this limitation retryLimit = retryLimit or self.retryLimit if self._useZip: self.removeDir(remoteDir) - self.mkDirs(remoteDir+"/x") + self.mkDirs(remoteDir + "/x") try: localZip = tempfile.mktemp() + ".zip" remoteZip = remoteDir + "/adbdmtmp.zip" proc = ProcessHandler(["zip", "-r", localZip, '.'], cwd=localDir, - processOutputLine=self._log) + processOutputLine=self._log) proc.run() proc.wait() self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False) mozfile.remove(localZip) data = self._runCmd(["shell", "unzip", "-o", remoteZip, "-d", remoteDir]).output[0] self._checkCmd(["shell", "rm", remoteZip], retryLimit=retryLimit, timeout=self.short_timeout) @@ -268,17 +269,17 @@ class DeviceManagerADB(DeviceManager): self._logger.warning(traceback.format_exc()) self._logger.warning("zip/unzip failure: falling back to normal push") self._useZip = False self.pushDir(localDir, remoteDir, retryLimit=retryLimit, timeout=timeout) else: # If the remote directory exists, newer implementations of # "adb push" will create a sub-directory, while older versions # will not! Bug 1285040 - self.mkDirs(remoteDir+"/x") + self.mkDirs(remoteDir + "/x") self.removeDir(remoteDir) tmpDir = tempfile.mkdtemp() # copytree's target dir must not already exist, so create a subdir tmpDirTarget = os.path.join(tmpDir, "tmp") shutil.copytree(localDir, tmpDirTarget) self._checkCmd(["push", tmpDirTarget, remoteDir], retryLimit=retryLimit, timeout=timeout) mozfile.remove(tmpDir) @@ -360,23 +361,23 @@ class DeviceManagerADB(DeviceManager): def fireProcess(self, appname, failIfRunning=False): """ Starts a process returns: pid DEPRECATED: Use shell() or launchApplication() for new code """ - #strip out env vars - parts = appname.split('"'); + # strip out env vars + parts = appname.split('"') if (len(parts) > 2): parts = parts[2:] return self.launchProcess(parts, failIfRunning) - def launchProcess(self, cmd, outputFile = "process.txt", cwd = '', env = '', failIfRunning=False): + def launchProcess(self, cmd, outputFile="process.txt", cwd='', env='', failIfRunning=False): """ Launches a process, redirecting output to standard out WARNING: Does not work how you expect on Android! The application's own output will be flushed elsewhere. DEPRECATED: Use shell() or launchApplication() for new code """ @@ -398,23 +399,23 @@ class DeviceManagerADB(DeviceManager): else: args = cmd[i:].strip() acmd.append("-n") acmd.append(cmd[0:i] + "/org.mozilla.gecko.BrowserApp") if args != "": acmd.append("--es") acmd.append("args") acmd.append(args) - if env != '' and env != None: + if env != '' and env is not None: envCnt = 0 # env is expected to be a dict of environment variables for envkey, envval in env.iteritems(): acmd.append("--es") acmd.append("env" + str(envCnt)) - acmd.append(envkey + "=" + envval); + acmd.append(envkey + "=" + envval) envCnt += 1 if uri != "": acmd.append("-d") acmd.append(uri) acmd = ["shell", ' '.join(map(lambda x: '"' + x + '"', ["am", "start"] + acmd))] self._logger.info(acmd) self._checkCmd(acmd) @@ -499,51 +500,52 @@ class DeviceManagerADB(DeviceManager): root = os.path.join(basePath, subPath) try: self.mkDir(root) return root except: pass raise DMError("Unable to set up device root using paths: [%s]" - % ", ".join(["'%s'" % os.path.join(b, s) for b, s in paths])) + % ", ".join(["'%s'" % os.path.join(b, s) for b, s in paths])) def getTempDir(self): # Cache result to speed up operations depending # on the temporary directory. if not self._tempDir: self._tempDir = "%s/tmp" % self.deviceRoot self.mkDir(self._tempDir) return self._tempDir - def reboot(self, wait = False, **kwargs): + def reboot(self, wait=False, **kwargs): self._checkCmd(["reboot"]) if wait: self._checkCmd(["wait-for-device"]) if self._runAdbAsRoot: self._adb_root() self._checkCmd(["shell", "ls", "/sbin"], timeout=self.short_timeout) def updateApp(self, appBundlePath, **kwargs): return self._runCmd(["install", "-r", appBundlePath]).output def getCurrentTime(self): timestr = str(self._runCmd(["shell", "date", "+%s"], timeout=self.short_timeout).output[0]) if (not timestr or not timestr.isdigit()): raise DMError("Unable to get current time using date (got: '%s')" % timestr) - return int(timestr)*1000 + return int(timestr) * 1000 def getInfo(self, directive=None): directive = directive or "all" ret = {} if directive == "id" or directive == "all": ret["id"] = self._runCmd(["get-serialno"], timeout=self.short_timeout).output[0] if directive == "os" or directive == "all": - ret["os"] = self.shellCheckOutput(["getprop", "ro.build.display.id"], timeout=self.short_timeout) + ret["os"] = self.shellCheckOutput( + ["getprop", "ro.build.display.id"], timeout=self.short_timeout) if directive == "uptime" or directive == "all": uptime = self.shellCheckOutput(["uptime"], timeout=self.short_timeout) if not uptime: raise DMError("error getting uptime") m = re.match("up time: ((\d+) days, )*(\d{2}):(\d{2}):(\d{2})", uptime) if m: uptime = "%d days %d hours %d minutes %d seconds" % tuple( [int(g or 0) for g in m.groups()[1:]]) @@ -555,17 +557,18 @@ class DeviceManagerADB(DeviceManager): ret["systime"] = self.shellCheckOutput(["date"], timeout=self.short_timeout) if directive == "memtotal" or directive == "all": meminfo = {} for line in self.pullFile("/proc/meminfo").splitlines(): key, value = line.split(":") meminfo[key] = value.strip() ret["memtotal"] = meminfo["MemTotal"] if directive == "disk" or directive == "all": - data = self.shellCheckOutput(["df", "/data", "/system", "/sdcard"], timeout=self.short_timeout) + data = self.shellCheckOutput( + ["df", "/data", "/system", "/sdcard"], timeout=self.short_timeout) ret["disk"] = data.split('\n') self._logger.debug("getInfo: %s" % ret) return ret def uninstallApp(self, appName, installPath=None): status = self._runCmd(["uninstall", appName]).output[0].strip() if status != 'Success': raise DMError("uninstall failed for %s. Got: %s" % (appName, status)) @@ -595,20 +598,20 @@ class DeviceManagerADB(DeviceManager): timeout = self.default_timeout def _timeout(): self._logger.error("Timeout exceeded for _runCmd call '%s'" % ' '.join(finalArgs)) retries = 0 while retries < retryLimit: proc = ProcessHandler(finalArgs, storeOutput=True, - processOutputLine=self._log, onTimeout=_timeout) + processOutputLine=self._log, onTimeout=_timeout) proc.run(timeout=timeout) proc.returncode = proc.wait() - if proc.returncode == None: + if proc.returncode is None: proc.kill() retries += 1 else: return proc # timeout is specified in seconds, and if no timeout is given, # we will run until we hit the default_timeout specified in the __init__ def _checkCmd(self, args, timeout=None, retryLimit=None): @@ -637,17 +640,17 @@ class DeviceManagerADB(DeviceManager): self._logger.error("Timeout exceeded for _checkCmd call '%s'" % ' '.join(finalArgs)) timeout = int(timeout) retries = 0 while retries < retryLimit: proc = ProcessHandler(finalArgs, processOutputLine=self._log, onTimeout=_timeout) proc.run(timeout=timeout) ret_code = proc.wait() - if ret_code == None: + if ret_code is None: proc.kill() retries += 1 else: return ret_code raise DMError("Timeout exceeded for _checkCmd call after %d retries." % retries) def chmodDir(self, remoteDir, mask="777"): @@ -656,17 +659,18 @@ class DeviceManagerADB(DeviceManager): self._logger.debug("chmod %s -- skipped (/sdcard)" % remoteDir) else: files = self.listFiles(remoteDir.strip()) for f in files: remoteEntry = remoteDir.strip() + "/" + f.strip() if (self.dirExists(remoteEntry)): self.chmodDir(remoteEntry) else: - self._checkCmd(["shell", "chmod", mask, remoteEntry], timeout=self.short_timeout) + self._checkCmd(["shell", "chmod", mask, remoteEntry], + timeout=self.short_timeout) self._logger.info("chmod %s" % remoteEntry) self._checkCmd(["shell", "chmod", mask, remoteDir], timeout=self.short_timeout) self._logger.debug("chmod %s" % remoteDir) else: self._checkCmd(["shell", "chmod", mask, remoteDir.strip()], timeout=self.short_timeout) self._logger.debug("chmod %s" % remoteDir.strip()) def _verifyADB(self): @@ -675,28 +679,30 @@ class DeviceManagerADB(DeviceManager): """ if self._adbPath != 'adb': if not os.access(self._adbPath, os.X_OK): raise DMError("invalid adb path, or adb not executable: %s" % self._adbPath) try: self._checkCmd(["version"], timeout=self.short_timeout) except os.error as err: - raise DMError("unable to execute ADB (%s): ensure Android SDK is installed and adb is in your $PATH" % err) + raise DMError( + "unable to execute ADB (%s): ensure Android SDK is installed " + "and adb is in your $PATH" % err) def _verifyDevice(self): # If there is a device serial number, see if adb is connected to it if self._deviceSerial: deviceStatus = None for line in self._runCmd(["devices"]).output: m = re.match('(.+)?\s+(.+)$', line) if m: if self._deviceSerial == m.group(1): deviceStatus = m.group(2) - if deviceStatus == None: + if deviceStatus is None: raise DMError("device not found: %s" % self._deviceSerial) elif deviceStatus != "device": raise DMError("bad status for device %s: %s" % (self._deviceSerial, deviceStatus)) # Check to see if we can connect to device and run a simple command if not self._checkCmd(["shell", "echo"], timeout=self.short_timeout) == 0: raise DMError("unable to connect to device") @@ -725,17 +731,17 @@ class DeviceManagerADB(DeviceManager): # wait for response for maximum of 15 seconds, in case su # prompts for a password or triggers the Android SuperUser # prompt start_time = time.time() retcode = None while (time.time() - start_time) <= 15 and retcode is None: retcode = proc.poll() - if retcode is None: # still not terminated, kill + if retcode is None: # still not terminated, kill proc.kill() if proc.output and 'uid=0(root)' in proc.output[0]: return True return False if su_id('0', self.short_timeout): self._haveSu = True @@ -784,19 +790,18 @@ class DeviceManagerADB(DeviceManager): if self.processInfo("adbd")[2] != "root": raise DMError("We tried rebooting adbd as root, however, it failed.") def _detectLsModifier(self): if self._lsModifier is None: # Check if busybox -1A is required in order to get one # file per line. output = self._runCmd(["shell", "ls", "-1A", "/"], - timeout=self.short_timeout).output + timeout=self.short_timeout).output output = ' '.join(output) if 'error: device not found' in output: raise DMError(output) if "Unknown option '-1'. Aborting." in output: self._lsModifier = "-a" elif "No such file or directory" in output: self._lsModifier = "-a" else: self._lsModifier = "-1A" -
--- a/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py +++ b/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py @@ -12,16 +12,17 @@ import os import re import posixpath import subprocess import StringIO from devicemanager import DeviceManager, DMError, _pop_last_line import errno from distutils.version import StrictVersion + class DeviceManagerSUT(DeviceManager): """ Implementation of DeviceManager interface that speaks to a device over TCP/IP using the "system under test" protocol. A software agent such as Negatus (http://github.com/mozilla/Negatus) or the Mozilla Android SUTAgent app must be present and listening for connections for this to work. """ @@ -40,17 +41,17 @@ class DeviceManagerSUT(DeviceManager): deviceRoot=deviceRoot) self.host = host self.port = port self.retryLimit = retryLimit self._sock = None self._everConnected = False # Get version - verstring = self._runCmds([{ 'cmd': 'ver' }]) + verstring = self._runCmds([{'cmd': 'ver'}]) ver_re = re.match('(\S+) Version (\S+)', verstring) self.agentProductName = ver_re.group(1) self.agentVersion = ver_re.group(2) def _cmdNeedsResponse(self, cmd): """ Not all commands need a response from the agent: * rebt obviously doesn't get a response * uninstall performs a reboot to ensure starting in a clean state and @@ -104,17 +105,17 @@ class DeviceManagerSUT(DeviceManager): re.compile('^rebt.*'), re.compile('^uninst .*$')] for c in socketClosingCmds: if (c.match(cmd)): return True return False - def _sendCmds(self, cmdlist, outputfile, timeout = None, retryLimit = None): + def _sendCmds(self, cmdlist, outputfile, timeout=None, retryLimit=None): """ Wrapper for _doCmds that loops up to retryLimit iterations """ # this allows us to move the retry logic outside of the _doCmds() to make it # easier for debugging in the future. # note that since cmdlist is a list of commands, they will all be retried if # one fails. this is necessary in particular for pushFile(), where we don't want # to accidentally send extra data if a failure occurs during data transmission. @@ -133,61 +134,65 @@ class DeviceManagerSUT(DeviceManager): self._logger.debug(err) retries += 1 # if we lost the connection or failed to establish one, wait a bit if retries < retryLimit and not self._sock: sleep_time = 5 * retries self._logger.info('Could not connect; sleeping for %d seconds.' % sleep_time) time.sleep(sleep_time) - raise DMError("Remote Device Error: unable to connect to %s after %s attempts" % (self.host, retryLimit)) + raise DMError("Remote Device Error: unable to connect to %s after %s attempts" % + (self.host, retryLimit)) - def _runCmds(self, cmdlist, timeout = None, retryLimit = None): + def _runCmds(self, cmdlist, timeout=None, retryLimit=None): """ Similar to _sendCmds, but just returns any output as a string instead of writing to a file """ retryLimit = retryLimit or self.retryLimit outputfile = StringIO.StringIO() self._sendCmds(cmdlist, outputfile, timeout, retryLimit=retryLimit) outputfile.seek(0) return outputfile.read() def _doCmds(self, cmdlist, outputfile, timeout): promptre = re.compile(self._prompt_regex + '$') shouldCloseSocket = False if not timeout: - # We are asserting that all commands will complete in this time unless otherwise specified + # We are asserting that all commands will complete in this time unless + # otherwise specified timeout = self.default_timeout if not self._sock: try: if self._everConnected: self._logger.info("reconnecting socket") self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) except socket.error as msg: self._sock = None - raise DMError("Automation Error: unable to create socket: "+str(msg)) + raise DMError("Automation Error: unable to create socket: " + str(msg)) try: self._sock.settimeout(float(timeout)) self._sock.connect((self.host, int(self.port))) self._everConnected = True except socket.error as msg: self._sock = None - raise DMError("Remote Device Error: Unable to connect socket: "+str(msg)) + raise DMError("Remote Device Error: Unable to connect socket: " + str(msg)) # consume prompt try: self._sock.recv(1024) except socket.error as msg: self._sock.close() self._sock = None - raise DMError("Remote Device Error: Did not get prompt after connecting: " + str(msg), fatal=True) + raise DMError( + "Remote Device Error: Did not get prompt after connecting: " + str(msg), + fatal=True) # future recv() timeouts are handled by select() calls self._sock.settimeout(None) for cmd in cmdlist: cmdline = '%s\r\n' % cmd['cmd'] try: @@ -203,18 +208,18 @@ class DeviceManagerSUT(DeviceManager): if sent == 0: raise DMError("Socket connection broken when sending data") totalsent += sent self._logger.debug("sent cmd: %s" % cmd['cmd']) except socket.error as msg: self._sock.close() self._sock = None - self._logger.error("Remote Device Error: Error sending data"\ - " to socket. cmd=%s; err=%s" % (cmd['cmd'], msg)) + self._logger.error("Remote Device Error: Error sending data" + " to socket. cmd=%s; err=%s" % (cmd['cmd'], msg)) return False # Check if the command should close the socket shouldCloseSocket = self._shouldCmdCloseSocket(cmd['cmd']) # Handle responses from commands if self._cmdNeedsResponse(cmd['cmd']): foundPrompt = False @@ -238,28 +243,31 @@ class DeviceManagerSUT(DeviceManager): timer = 0 if not temp: socketClosed = True errStr = 'connection closed' timer += select_timeout if timer > timeout: self._sock.close() self._sock = None - raise DMError("Automation Error: Timeout in command %s" % cmd['cmd'], fatal=True) + raise DMError("Automation Error: Timeout in command %s" % + cmd['cmd'], fatal=True) except socket.error as err: socketClosed = True errStr = str(err) # This error shows up with we have our tegra rebooted. if err[0] == errno.ECONNRESET: errStr += ' - possible reboot' if socketClosed: self._sock.close() self._sock = None - raise DMError("Automation Error: Error receiving data from socket. cmd=%s; err=%s" % (cmd, errStr)) + raise DMError( + "Automation Error: Error receiving data from socket. " + "cmd=%s; err=%s" % (cmd, errStr)) data += temp # If something goes wrong in the agent it will send back a string that # starts with '##AGENT-WARNING##' if not commandFailed: errorMatch = self._agentErrorRE.match(data) if errorMatch: @@ -271,18 +279,18 @@ class DeviceManagerSUT(DeviceManager): if promptre.match(line): foundPrompt = True data = self._stripPrompt(data) break # periodically flush data to output file to make sure it doesn't get # too big/unwieldly if len(data) > 1024: - outputfile.write(data[0:1024]) - data = data[1024:] + outputfile.write(data[0:1024]) + data = data[1024:] if commandFailed: raise DMError("Automation Error: Error processing command '%s'; err='%s'" % (cmd['cmd'], errorMatch.group(1)), fatal=True) # Write any remaining data to outputfile outputfile.write(data) @@ -292,17 +300,17 @@ class DeviceManagerSUT(DeviceManager): self._sock = None except: self._sock = None raise DMError("Automation Error: Error closing socket") def _setupDeviceRoot(self, deviceRoot): if not deviceRoot: deviceRoot = "%s/tests" % self._runCmds( - [{ 'cmd': 'testroot' }]).strip() + [{'cmd': 'testroot'}]).strip() self.mkDir(deviceRoot) return deviceRoot def shell(self, cmd, outputfile, env=None, cwd=None, timeout=None, root=False): cmdline = self._escapedCommandLine(cmd) if env: cmdline = '%s %s' % (self._formatEnvString(env), cmdline) @@ -322,62 +330,63 @@ class DeviceManagerSUT(DeviceManager): cmd = "exec" if cwd: cmd += "cwd" if root and haveExecSu: cmd += "su" if cwd: - self._sendCmds([{ 'cmd': '%s %s %s' % (cmd, cwd, cmdline) }], outputfile, timeout) + self._sendCmds([{'cmd': '%s %s %s' % (cmd, cwd, cmdline)}], outputfile, timeout) else: if (not root) or haveExecSu: - self._sendCmds([{ 'cmd': '%s %s' % (cmd, cmdline) }], outputfile, timeout) + self._sendCmds([{'cmd': '%s %s' % (cmd, cmdline)}], outputfile, timeout) else: # need to manually inject su -c for backwards compatibility (this may # not work on ICS or above!!) # (FIXME: this backwards compatibility code is really ugly and should # be deprecated at some point in the future) - self._sendCmds([ { 'cmd': '%s su -c "%s"' % (cmd, cmdline) }], outputfile, + self._sendCmds([{'cmd': '%s su -c "%s"' % (cmd, cmdline)}], outputfile, timeout) # dig through the output to get the return code lastline = _pop_last_line(outputfile) if lastline: m = re.search('return code \[([0-9]+)\]', lastline) if m: return int(m.group(1)) # woops, we couldn't find an end of line/return value - raise DMError("Automation Error: Error finding end of line/return value when running '%s'" % cmdline) + raise DMError( + "Automation Error: Error finding end of line/return value when running '%s'" % cmdline) def pushFile(self, localname, destname, retryLimit=None, createDir=True): retryLimit = retryLimit or self.retryLimit if createDir: self.mkDirs(destname) try: filesize = os.path.getsize(localname) with open(localname, 'rb') as f: - remoteHash = self._runCmds([{ 'cmd': 'push ' + destname + ' ' + str(filesize), - 'data': f.read() }], retryLimit=retryLimit).strip() + remoteHash = self._runCmds([{'cmd': 'push ' + destname + ' ' + str(filesize), + 'data': f.read()}], retryLimit=retryLimit).strip() except OSError: raise DMError("DeviceManager: Error reading file to push") self._logger.debug("push returned: %s" % remoteHash) localHash = self._getLocalHash(localname) if localHash != remoteHash: raise DMError("Automation Error: Push File failed to Validate! (localhash: %s, " "remotehash: %s)" % (localHash, remoteHash)) def mkDir(self, name): if not self.dirExists(name): - self._runCmds([{ 'cmd': 'mkdr ' + name }]) + self._runCmds([{'cmd': 'mkdr ' + name}]) def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None): retryLimit = retryLimit or self.retryLimit self._logger.info("pushing directory: %s to %s" % (localDir, remoteDir)) existentDirectories = [] for root, dirs, files in os.walk(localDir, followlinks=True): _, subpath = root.split(localDir) @@ -389,20 +398,21 @@ class DeviceManagerSUT(DeviceManager): if subpath == "": remoteRoot = remoteDir parent = os.path.dirname(remoteName) if parent not in existentDirectories: self.mkDirs(remoteName) existentDirectories.append(parent) - self.pushFile(os.path.join(root, f), remoteName, retryLimit=retryLimit, createDir=False) + self.pushFile(os.path.join(root, f), remoteName, + retryLimit=retryLimit, createDir=False) def dirExists(self, remotePath): - ret = self._runCmds([{ 'cmd': 'isdir ' + remotePath }]).strip() + ret = self._runCmds([{'cmd': 'isdir ' + remotePath}]).strip() if not ret: raise DMError('Automation Error: DeviceManager isdir returned null') return ret == 'TRUE' def fileExists(self, filepath): # Because we always have / style paths we make this a lot easier with some # assumptions @@ -413,42 +423,42 @@ class DeviceManagerSUT(DeviceManager): return self.dirExists(filepath) (containingpath, filename) = posixpath.split(filepath) return filename in self.listFiles(containingpath) def listFiles(self, rootdir): rootdir = posixpath.normpath(rootdir) if not self.dirExists(rootdir): return [] - data = self._runCmds([{ 'cmd': 'cd ' + rootdir }, { 'cmd': 'ls' }]) + data = self._runCmds([{'cmd': 'cd ' + rootdir}, {'cmd': 'ls'}]) files = filter(lambda x: x, data.splitlines()) if len(files) == 1 and files[0] == '<empty>': # special case on the agent: empty directories return just the # string "<empty>" return [] return files def removeFile(self, filename): self._logger.info("removing file: " + filename) if self.fileExists(filename): - self._runCmds([{ 'cmd': 'rm ' + filename }]) + self._runCmds([{'cmd': 'rm ' + filename}]) def removeDir(self, remoteDir): if self.dirExists(remoteDir): - self._runCmds([{ 'cmd': 'rmdr ' + remoteDir }]) + self._runCmds([{'cmd': 'rmdr ' + remoteDir}]) def moveTree(self, source, destination): - self._runCmds([{ 'cmd': 'mv %s %s' % (source, destination) }]) + self._runCmds([{'cmd': 'mv %s %s' % (source, destination)}]) def copyTree(self, source, destination): - self._runCmds([{ 'cmd': 'dd if=%s of=%s' % (source, destination) }]) + self._runCmds([{'cmd': 'dd if=%s of=%s' % (source, destination)}]) def getProcessList(self): - data = self._runCmds([{ 'cmd': 'ps' }]) + data = self._runCmds([{'cmd': 'ps'}]) processTuples = [] for line in data.splitlines(): if line: pidproc = line.strip().split() try: if (len(pidproc) == 2): processTuples += [[pidproc[0], pidproc[1]]] @@ -473,22 +483,22 @@ class DeviceManagerSUT(DeviceManager): DEPRECATED: Use shell() or launchApplication() for new code """ if not appname: raise DMError("Automation Error: fireProcess called with no command to run") self._logger.info("FIRE PROC: '%s'" % appname) - if (self.processExist(appname) != None): + if (self.processExist(appname) is None): self._logger.warning("process %s appears to be running already\n" % appname) if (failIfRunning): raise DMError("Automation Error: Process is already running") - self._runCmds([{ 'cmd': 'exec ' + appname }]) + self._runCmds([{'cmd': 'exec ' + appname}]) # The 'exec' command may wait for the process to start and end, so checking # for the process here may result in process = None. # The normal case is to launch the process and return right away # There is one case with robotium (am instrument) where exec returns at the end pid = None waited = 0 while pid is None and waited < maxWaitTime: @@ -496,17 +506,17 @@ class DeviceManagerSUT(DeviceManager): if pid: break time.sleep(1) waited += 1 self._logger.debug("got pid: %s for process: %s" % (pid, appname)) return pid - def launchProcess(self, cmd, outputFile = "process.txt", cwd = '', env = '', failIfRunning=False): + def launchProcess(self, cmd, outputFile="process.txt", cwd='', env='', failIfRunning=False): """ Launches a process, redirecting output to standard out Returns output filename WARNING: Does not work how you expect on Android! The application's own output will be flushed elsewhere. @@ -540,42 +550,42 @@ class DeviceManagerSUT(DeviceManager): return outputFile def killProcess(self, appname, sig=None): if sig: pid = self.processExist(appname) if pid and pid > 0: try: self.shellCheckOutput(['kill', '-%d' % sig, str(pid)], - root=True) + root=True) except DMError as err: self._logger.warning("unable to kill -%d %s (pid %s)" % - (sig, appname, str(pid))) + (sig, appname, str(pid))) self._logger.debug(err) raise err else: self._logger.warning("unable to kill -%d %s -- not running?" % - (sig, appname)) + (sig, appname)) else: retries = 0 while retries < self.retryLimit: try: if self.processExist(appname): - self._runCmds([{ 'cmd': 'kill ' + appname }]) + self._runCmds([{'cmd': 'kill ' + appname}]) return except DMError as err: retries += 1 self._logger.warning("try %d of %d failed to kill %s" % - (retries, self.retryLimit, appname)) + (retries, self.retryLimit, appname)) self._logger.debug(err) if retries >= self.retryLimit: raise err def getTempDir(self): - return self._runCmds([{ 'cmd': 'tmpd' }]).strip() + return self._runCmds([{'cmd': 'tmpd'}]).strip() def pullFile(self, remoteFile, offset=None, length=None): # The "pull" command is different from other commands in that DeviceManager # has to read a certain number of bytes instead of just reading to the # next prompt. This is more robust than the "cat" command, which will be # confused if the prompt string exists within the file being catted. # However it means we can't use the response-handling logic in sendCMD(). @@ -600,17 +610,17 @@ class DeviceManagerSUT(DeviceManager): err(error_msg) return data except: err(error_msg) def read_until_char(c, buf, error_msg): """ read until 'c' is found; buffer rest """ - while not c in buf: + while c not in buf: data = uread(1024, error_msg) buf += data return buf.partition(c) def read_exact(total_to_recv, buf, error_msg): """ read exact number of 'total_to_recv' bytes """ while len(buf) < total_to_recv: to_recv = min(total_to_recv - len(buf), 1024) @@ -626,20 +636,20 @@ class DeviceManagerSUT(DeviceManager): # or, if error, # <filename>,-1\n<error message> # just send the command first, we read the response inline below if offset is not None and length is not None: cmd = 'pull %s %d %d' % (remoteFile, offset, length) elif offset is not None: cmd = 'pull %s %d' % (remoteFile, offset) - else: + else: cmd = 'pull %s' % remoteFile - self._runCmds([{ 'cmd': cmd }]) + self._runCmds([{'cmd': cmd}]) # read metadata; buffer the rest metadata, sep, buf = read_until_char('\n', buf, 'could not find metadata') if not metadata: return None self._logger.debug('metadata: %s' % metadata) filename, sep, filesizestr = metadata.partition(',') @@ -653,17 +663,18 @@ class DeviceManagerSUT(DeviceManager): if filesize == -1: # read error message error_str, sep, buf = read_until_char('\n', buf, 'could not find error message') if not error_str: err("blank error message") # prompt should follow read_exact(len(prompt), buf, 'could not find prompt') # failures are expected, so don't use "Remote Device Error" or we'll RETRY - raise DMError("DeviceManager: pulling file '%s' unsuccessful: %s" % (remoteFile, error_str)) + raise DMError("DeviceManager: pulling file '%s' unsuccessful: %s" % + (remoteFile, error_str)) # read file data total_to_recv = filesize + len(prompt) buf = read_exact(total_to_recv, buf, 'could not get all file data') if buf[-len(prompt):] != prompt: err('no prompt found after file data--DeviceManager may be out of sync with agent') return buf return buf[:-len(prompt)] @@ -698,43 +709,43 @@ class DeviceManagerSUT(DeviceManager): self.getDirectory(remotePath, localPath, False) else: self.getFile(remotePath, localPath) def validateFile(self, remoteFile, localFile): remoteHash = self._getRemoteHash(remoteFile) localHash = self._getLocalHash(localFile) - if (remoteHash == None): + if (remoteHash is None): return False if (remoteHash == localHash): return True return False def _getRemoteHash(self, filename): - data = self._runCmds([{ 'cmd': 'hash ' + filename }]).strip() + data = self._runCmds([{'cmd': 'hash ' + filename}]).strip() self._logger.debug("remote hash returned: '%s'" % data) return data def unpackFile(self, filePath, destDir=None): """ Unzips a bundle to a location on the device If destDir is not specified, the bundle is extracted in the same directory """ # if no destDir is passed in just set it to filePath's folder if not destDir: destDir = posixpath.dirname(filePath) if destDir[-1] != '/': destDir += '/' - self._runCmds([{ 'cmd': 'unzp %s %s' % (filePath, destDir)}]) + self._runCmds([{'cmd': 'unzp %s %s' % (filePath, destDir)}]) def _getRebootServerSocket(self, ipAddr): serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) serverSocket.settimeout(60.0) serverSocket.bind((ipAddr, 0)) serverSocket.listen(1) self._logger.debug('Created reboot callback server at %s:%d' % @@ -765,17 +776,16 @@ class DeviceManagerSUT(DeviceManager): if not data: raise DMError('Timed out waiting for reboot callback.') self._logger.info("Sleeping for %s seconds to wait for device " "to 'settle'" % self.reboot_settling_time) time.sleep(self.reboot_settling_time) - def reboot(self, ipAddr=None, port=30000, wait=False): # port ^^^ is here for backwards compatibility only, we now # determine a port automatically and safely wait = (wait or ipAddr) cmd = 'rebt' self._logger.info("Rebooting device") @@ -804,24 +814,24 @@ class DeviceManagerSUT(DeviceManager): if wait: self._waitForRebootPing(serverSocket) def getInfo(self, directive=None): data = None result = {} collapseSpaces = re.compile(' +') - directives = ['os','id','uptime','uptimemillis','systime','screen', - 'rotation','memory','process','disk','power','sutuserinfo', + directives = ['os', 'id', 'uptime', 'uptimemillis', 'systime', 'screen', + 'rotation', 'memory', 'process', 'disk', 'power', 'sutuserinfo', 'temperature'] if (directive in directives): directives = [directive] for d in directives: - data = self._runCmds([{ 'cmd': 'info ' + d }]) + data = self._runCmds([{'cmd': 'info ' + d}]) data = collapseSpaces.sub(' ', data) result[d] = data.split('\n') # Get rid of any 0 length members of the arrays for k, v in result.iteritems(): result[k] = filter(lambda x: x != '', result[k]) @@ -836,38 +846,38 @@ class DeviceManagerSUT(DeviceManager): self._logger.debug("results: %s" % result) return result def installApp(self, appBundlePath, destPath=None): cmd = 'inst ' + appBundlePath if destPath: cmd += ' ' + destPath - data = self._runCmds([{ 'cmd': cmd }]) + data = self._runCmds([{'cmd': cmd}]) if 'installation complete [0]' not in data: raise DMError("Remove Device Error: Error installing app. Error message: %s" % data) def uninstallApp(self, appName, installPath=None): cmd = 'uninstall ' + appName if installPath: cmd += ' ' + installPath - data = self._runCmds([{ 'cmd': cmd }]) + data = self._runCmds([{'cmd': cmd}]) status = data.split('\n')[0].strip() self._logger.debug("uninstallApp: '%s'" % status) if status == 'Success': return raise DMError("Remote Device Error: uninstall failed for %s" % appName) def uninstallAppAndReboot(self, appName, installPath=None): cmd = 'uninst ' + appName if installPath: cmd += ' ' + installPath - data = self._runCmds([{ 'cmd': cmd }]) + data = self._runCmds([{'cmd': cmd}]) self._logger.debug("uninstallAppAndReboot: %s" % data) return def updateApp(self, appBundlePath, processName=None, destPath=None, ipAddr=None, port=30000, wait=False): # port ^^^ is here for backwards compatibility only, we now # determine a port automatically and safely @@ -892,69 +902,74 @@ class DeviceManagerSUT(DeviceManager): self._logger.debug("updateApp using command: " % cmd) self._runCmds([{'cmd': cmd}]) if wait: self._waitForRebootPing(serverSocket) def getCurrentTime(self): - return int(self._runCmds([{ 'cmd': 'clok' }]).strip()) + return int(self._runCmds([{'cmd': 'clok'}]).strip()) def _formatEnvString(self, env): """ Returns a properly formatted env string for the agent. Input - env, which is either None, '', or a dict Output - a quoted string of the form: '"envvar1=val1,envvar2=val2..."' If env is None or '' return '' (empty quoted string) """ - if (env == None or env == ''): + if (env is None or env == ''): return '' retVal = '"%s"' % ','.join(map(lambda x: '%s=%s' % (x[0], x[1]), env.iteritems())) if (retVal == '""'): return '' return retVal def adjustResolution(self, width=1680, height=1050, type='hdmi'): """ Adjust the screen resolution on the device, REBOOT REQUIRED NOTE: this only works on a tegra ATM - supported resolutions: 640x480, 800x600, 1024x768, 1152x864, 1200x1024, 1440x900, 1680x1050, 1920x1080 + supported resolutions: 640x480, 800x600, 1024x768, 1152x864, 1200x1024, 1440x900, + 1680x1050, 1920x1080 """ if self.getInfo('os')['os'][0].split()[0] != 'harmony-eng': self._logger.warning("unable to adjust screen resolution on non Tegra device") return False results = self.getInfo('screen') parts = results['screen'][0].split(':') - self._logger.debug("we have a current resolution of %s, %s" % (parts[1].split()[0], parts[2].split()[0])) + self._logger.debug("we have a current resolution of %s, %s" % + (parts[1].split()[0], parts[2].split()[0])) - #verify screen type is valid, and set it to the proper value (https://bugzilla.mozilla.org/show_bug.cgi?id=632895#c4) + # verify screen type is valid, and set it to the proper value + # (https://bugzilla.mozilla.org/show_bug.cgi?id=632895#c4) screentype = -1 if (type == 'hdmi'): screentype = 5 elif (type == 'vga' or type == 'crt'): screentype = 3 else: return False - #verify we have numbers + # verify we have numbers if not (isinstance(width, int) and isinstance(height, int)): return False if (width < 100 or width > 9999): return False if (height < 100 or height > 9999): return False self._logger.debug("adjusting screen resolution to %s, %s and rebooting" % (width, height)) - self._runCmds([{ 'cmd': "exec setprop persist.tegra.dpy%s.mode.width %s" % (screentype, width) }]) - self._runCmds([{ 'cmd': "exec setprop persist.tegra.dpy%s.mode.height %s" % (screentype, height) }]) + self._runCmds( + [{'cmd': "exec setprop persist.tegra.dpy%s.mode.width %s" % (screentype, width)}]) + self._runCmds( + [{'cmd': "exec setprop persist.tegra.dpy%s.mode.height %s" % (screentype, height)}]) def chmodDir(self, remoteDir, **kwargs): - self._runCmds([{ 'cmd': "chmod "+remoteDir }]) + self._runCmds([{'cmd': "chmod " + remoteDir}])
--- a/testing/mozbase/mozdevice/mozdevice/dmcli.py +++ b/testing/mozbase/mozdevice/mozdevice/dmcli.py @@ -11,131 +11,138 @@ import logging import os import posixpath import StringIO import sys import mozdevice import mozlog import argparse + class DMCli(object): def __init__(self): - self.commands = { 'deviceroot': { 'function': self.deviceroot, - 'help': 'get device root directory for storing temporary files' }, - 'install': { 'function': self.install, - 'args': [ { 'name': 'file' } ], - 'help': 'push this package file to the device and install it' }, - 'uninstall': { 'function': self.uninstall, - 'args': [ { 'name': 'packagename' } ], - 'help': 'uninstall the named app from the device' }, - 'killapp': { 'function': self.kill, - 'args': [ { 'name': 'process_name', 'nargs': '*' } ], - 'help': 'kills any processes with name(s) on device' }, - 'launchapp': { 'function': self.launchapp, - 'args': [ { 'name': 'appname' }, - { 'name': 'activity_name' }, - { 'name': '--intent', - 'action': 'store', - 'default': 'android.intent.action.VIEW' }, - { 'name': '--url', - 'action': 'store' }, - { 'name': '--no-fail-if-running', - 'action': 'store_true', - 'help': 'Don\'t fail if application is already running' } + self.commands = {'deviceroot': {'function': self.deviceroot, + 'help': 'get device root directory for storing temporary ' + 'files'}, + 'install': {'function': self.install, + 'args': [{'name': 'file'}], + 'help': 'push this package file to the device' + ' and install it'}, + 'uninstall': {'function': self.uninstall, + 'args': [{'name': 'packagename'}], + 'help': 'uninstall the named app from the device'}, + 'killapp': {'function': self.kill, + 'args': [{'name': 'process_name', 'nargs': '*'}], + 'help': 'kills any processes with name(s) on device'}, + 'launchapp': {'function': self.launchapp, + 'args': [{'name': 'appname'}, + {'name': 'activity_name'}, + {'name': '--intent', + 'action': 'store', + 'default': 'android.intent.action.VIEW'}, + {'name': '--url', + 'action': 'store'}, + {'name': '--no-fail-if-running', + 'action': 'store_true', + 'help': 'Don\'t fail if application is' + ' already running'} ], - 'help': 'launches application on device' }, - 'listapps': { 'function': self.listapps, - 'help': 'list applications on device' }, - 'push': { 'function': self.push, - 'args': [ { 'name': 'local_file' }, - { 'name': 'remote_file' } - ], - 'help': 'copy file/dir to device' }, - 'pull': { 'function': self.pull, - 'args': [ { 'name': 'local_file' }, - { 'name': 'remote_file', 'nargs': '?' } ], - 'help': 'copy file/dir from device' }, - 'shell': { 'function': self.shell, - 'args': [ { 'name': 'command', 'nargs': argparse.REMAINDER }, - { 'name': '--root', 'action': 'store_true', - 'help': 'Run command as root' }], - 'help': 'run shell command on device' }, - 'info': { 'function': self.getinfo, - 'args': [ { 'name': 'directive', 'nargs': '?' } ], - 'help': 'get information on specified ' - 'aspect of the device (if no argument ' - 'given, print all available information)' + 'help': 'launches application on device'}, + 'listapps': {'function': self.listapps, + 'help': 'list applications on device'}, + 'push': {'function': self.push, + 'args': [{'name': 'local_file'}, + {'name': 'remote_file'} + ], + 'help': 'copy file/dir to device'}, + 'pull': {'function': self.pull, + 'args': [{'name': 'local_file'}, + {'name': 'remote_file', 'nargs': '?'}], + 'help': 'copy file/dir from device'}, + 'shell': {'function': self.shell, + 'args': [{'name': 'command', 'nargs': argparse.REMAINDER}, + {'name': '--root', 'action': 'store_true', + 'help': 'Run command as root'}], + 'help': 'run shell command on device'}, + 'info': {'function': self.getinfo, + 'args': [{'name': 'directive', 'nargs': '?'}], + 'help': 'get information on specified ' + 'aspect of the device (if no argument ' + 'given, print all available information)' + }, + 'ps': {'function': self.processlist, + 'help': 'get information on running processes on device' + }, + 'logcat': {'function': self.logcat, + 'help': 'get logcat from device' }, - 'ps': { 'function': self.processlist, - 'help': 'get information on running processes on device' - }, - 'logcat' : { 'function': self.logcat, - 'help': 'get logcat from device' - }, - 'ls': { 'function': self.listfiles, - 'args': [ { 'name': 'remote_dir' } ], - 'help': 'list files on device' + 'ls': {'function': self.listfiles, + 'args': [{'name': 'remote_dir'}], + 'help': 'list files on device' }, - 'rm': { 'function': self.removefile, - 'args': [ { 'name': 'remote_file' } ], - 'help': 'remove file from device' - }, - 'isdir': { 'function': self.isdir, - 'args': [ { 'name': 'remote_dir' } ], - 'help': 'print if remote file is a directory' - }, - 'mkdir': { 'function': self.mkdir, - 'args': [ { 'name': 'remote_dir' } ], - 'help': 'makes a directory on device' + 'rm': {'function': self.removefile, + 'args': [{'name': 'remote_file'}], + 'help': 'remove file from device' }, - 'rmdir': { 'function': self.rmdir, - 'args': [ { 'name': 'remote_dir' } ], - 'help': 'recursively remove directory from device' - }, - 'screencap': { 'function': self.screencap, - 'args': [ { 'name': 'png_file' } ], - 'help': 'capture screenshot of device in action' - }, - 'sutver': { 'function': self.sutver, - 'help': 'SUTAgent\'s product name and version (SUT only)' + 'isdir': {'function': self.isdir, + 'args': [{'name': 'remote_dir'}], + 'help': 'print if remote file is a directory' + }, + 'mkdir': {'function': self.mkdir, + 'args': [{'name': 'remote_dir'}], + 'help': 'makes a directory on device' }, - 'clearlogcat': { 'function': self.clearlogcat, - 'help': 'clear the logcat' + 'rmdir': {'function': self.rmdir, + 'args': [{'name': 'remote_dir'}], + 'help': 'recursively remove directory from device' + }, + 'screencap': {'function': self.screencap, + 'args': [{'name': 'png_file'}], + 'help': 'capture screenshot of device in action' + }, + 'sutver': {'function': self.sutver, + 'help': 'SUTAgent\'s product name and version (SUT only)' + }, + 'clearlogcat': {'function': self.clearlogcat, + 'help': 'clear the logcat' }, - 'reboot': { 'function': self.reboot, - 'help': 'reboot the device', - 'args': [ { 'name': '--wait', - 'action': 'store_true', - 'help': 'Wait for device to come back up before exiting' } ] + 'reboot': {'function': self.reboot, + 'help': 'reboot the device', + 'args': [{'name': '--wait', + 'action': 'store_true', + 'help': 'Wait for device to come back up' + ' before exiting'}] - }, - 'isfile': { 'function': self.isfile, - 'args': [ { 'name': 'remote_file' } ], - 'help': 'check whether a file exists on the device' - }, - 'launchfennec': { 'function': self.launchfennec, - 'args': [ { 'name': 'appname' }, - { 'name': '--intent', 'action': 'store', - 'default': 'android.intent.action.VIEW' }, - { 'name': '--url', 'action': 'store' }, - { 'name': '--extra-args', 'action': 'store' }, - { 'name': '--mozenv', 'action': 'store', - 'help': 'Gecko environment variables to set in "KEY1=VAL1 KEY2=VAL2" format' }, - { 'name': '--no-fail-if-running', - 'action': 'store_true', - 'help': 'Don\'t fail if application is already running' } - ], - 'help': 'launch fennec' - }, - 'getip': { 'function': self.getip, - 'args': [ { 'name': 'interface', 'nargs': '*' } ], - 'help': 'get the ip address of the device' + }, + 'isfile': {'function': self.isfile, + 'args': [{'name': 'remote_file'}], + 'help': 'check whether a file exists on the device' + }, + 'launchfennec': {'function': self.launchfennec, + 'args': [{'name': 'appname'}, + {'name': '--intent', 'action': 'store', + 'default': 'android.intent.action.VIEW'}, + {'name': '--url', 'action': 'store'}, + {'name': '--extra-args', 'action': 'store'}, + {'name': '--mozenv', 'action': 'store', + 'help': 'Gecko environment variables to set' + ' in "KEY1=VAL1 KEY2=VAL2" format'}, + {'name': '--no-fail-if-running', + 'action': 'store_true', + 'help': 'Don\'t fail if application is ' + 'already running'} + ], + 'help': 'launch fennec' + }, + 'getip': {'function': self.getip, + 'args': [{'name': 'interface', 'nargs': '*'}], + 'help': 'get the ip address of the device' } - } + } self.parser = argparse.ArgumentParser() self.add_options(self.parser) self.add_commands(self.parser) mozlog.commandline.add_logging_group(self.parser) def run(self, args=sys.argv[1:]): args = self.parser.parse_args() @@ -157,28 +164,28 @@ class DMCli(object): sys.exit(ret) def add_options(self, parser): parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output from DeviceManager", default=bool(os.environ.get('VERBOSE'))) parser.add_argument("--host", action="store", - help="Device hostname (only if using TCP/IP, " \ - "defaults to TEST_DEVICE environment " \ - "variable if present)", + help="Device hostname (only if using TCP/IP, " + "defaults to TEST_DEVICE environment " + "variable if present)", default=os.environ.get('TEST_DEVICE')) parser.add_argument("-p", "--port", action="store", type=int, help="Custom device port (if using SUTAgent or " "adb-over-tcp)", default=None) parser.add_argument("-m", "--dmtype", action="store", - help="DeviceManager type (adb or sut, defaults " \ - "to DM_TRANS environment variable, if " \ - "present, or adb)", + help="DeviceManager type (adb or sut, defaults " + "to DM_TRANS environment variable, if " + "present, or adb)", default=os.environ.get('DM_TRANS', 'adb')) parser.add_argument("-d", "--hwid", action="store", help="HWID", default=None) parser.add_argument("--package-name", action="store", help="Packagename (if using DeviceManagerADB)", default=None) def add_commands(self, parser): @@ -360,15 +367,16 @@ class DMCli(object): failIfRunning=(not args.no_fail_if_running)) def getip(self, args): if args.interface: print(self.dm.getIP(args.interface)) else: print(self.dm.getIP()) + def cli(args=sys.argv[1:]): # process the command line cli = DMCli() cli.run(args) if __name__ == '__main__': cli()
--- a/testing/mozbase/mozdevice/mozdevice/droid.py +++ b/testing/mozbase/mozdevice/mozdevice/droid.py @@ -11,16 +11,17 @@ import time import version_codes from Zeroconf import Zeroconf, ServiceBrowser from devicemanager import ZeroconfListener from devicemanagerADB import DeviceManagerADB from devicemanagerSUT import DeviceManagerSUT from devicemanager import DMError + class DroidMixin(object): """Mixin to extend DeviceManager with Android-specific functionality""" _stopApplicationNeedsRoot = True def _getExtraAmStartArgs(self): return [] @@ -41,17 +42,17 @@ class DroidMixin(object): # If failIfRunning is True, we throw an exception here. Only one # instance of an application can be running at once on Android, # starting a new instance may not be what we want depending on what # we want to do if failIfRunning and self.processExist(appName): raise DMError("Only one instance of an application may be running " "at once") - acmd = [ "am", "start" ] + self._getExtraAmStartArgs() + \ + acmd = ["am", "start"] + self._getExtraAmStartArgs() + \ ["-W" if wait else '', "-n", "%s/%s" % (appName, activityName)] if intent: acmd.extend(["-a", intent]) if extras: for (key, val) in extras.iteritems(): if type(val) is int: @@ -98,17 +99,18 @@ class DroidMixin(object): for (envCnt, (envkey, envval)) in enumerate(mozEnv.iteritems()): extras["env" + str(envCnt)] = envkey + "=" + envval # Additional command line arguments that fennec will read and use (e.g. # with a custom profile) if extraArgs: extras['args'] = " ".join(extraArgs) - self.launchApplication(appName, "org.mozilla.gecko.BrowserApp", intent, url=url, extras=extras, + self.launchApplication(appName, "org.mozilla.gecko.BrowserApp", intent, url=url, + extras=extras, wait=wait, failIfRunning=failIfRunning) def getInstalledApps(self): """ Lists applications installed on this Android device Returns a list of application names in the form [ 'org.mozilla.fennec', ... ] """ @@ -129,51 +131,54 @@ class DroidMixin(object): we simply try to manually kill the processes started by the app repeatedly until none is around any more. This is less reliable and does require root. :param appName: Name of application (e.g. `com.android.chrome`) """ version = self.shellCheckOutput(["getprop", "ro.build.version.sdk"]) if int(version) >= version_codes.HONEYCOMB: - self.shellCheckOutput([ "am", "force-stop", appName ], root=self._stopApplicationNeedsRoot) + self.shellCheckOutput(["am", "force-stop", appName], + root=self._stopApplicationNeedsRoot) else: num_tries = 0 max_tries = 5 while self.processExist(appName): if num_tries > max_tries: raise DMError("Couldn't successfully kill %s after %s " "tries" % (appName, max_tries)) self.killProcess(appName) num_tries += 1 # sleep for a short duration to make sure there are no # additional processes in the process of being launched # (this is not 100% guaranteed to work since it is inherently # racey, but it's the best we can do) time.sleep(1) + class DroidADB(DeviceManagerADB, DroidMixin): _stopApplicationNeedsRoot = False def getTopActivity(self): package = None data = None try: - data = self.shellCheckOutput(["dumpsys", "window", "windows"], timeout=self.short_timeout) + data = self.shellCheckOutput( + ["dumpsys", "window", "windows"], timeout=self.short_timeout) except: # dumpsys seems to intermittently fail (seen on 4.3 emulator), producing # no output. return "" # "dumpsys window windows" produces many lines of input. The top/foreground # activity is indicated by something like: - # mFocusedApp=AppWindowToken{483e6db0 token=HistoryRecord{484dcad8 com.mozilla.SUTAgentAndroid/.SUTAgentAndroid}} + # mFocusedApp=AppWindowToken{483e6db0 token=HistoryRecord{484dcad8 com.mozilla.SUTAgentAndroid/.SUTAgentAndroid}} # noqa # or, on other devices: - # FocusedApplication: name='AppWindowToken{41a65340 token=ActivityRecord{418fbd68 org.mozilla.fennec_mozdev/org.mozilla.gecko.BrowserApp}}', dispatchingTimeout=5000.000ms + # FocusedApplication: name='AppWindowToken{41a65340 token=ActivityRecord{418fbd68 org.mozilla.fennec_mozdev/org.mozilla.gecko.BrowserApp}}', dispatchingTimeout=5000.000ms # noqa # Extract this line, ending in the forward slash: m = re.search('mFocusedApp(.+)/', data) if not m: m = re.search('FocusedApplication(.+)/', data) if m: line = m.group(0) # Extract package name: string of non-whitespace ending in forward slash m = re.search('(\S+)/$', line) @@ -189,50 +194,54 @@ class DroidADB(DeviceManagerADB, DroidMi def getAppRoot(self, packageName): """ Returns the root directory for the specified android application """ # relying on convention return '/data/data/%s' % packageName + class DroidSUT(DeviceManagerSUT, DroidMixin): def _getExtraAmStartArgs(self): # in versions of android in jellybean and beyond, the agent may run as # a different process than the one that started the app. In this case, # we need to get back the original user serial number and then pass # that to the 'am start' command line if not hasattr(self, '_userSerial'): infoDict = self.getInfo(directive="sutuserinfo") if infoDict.get('sutuserinfo') and \ len(infoDict['sutuserinfo']) > 0: - userSerialString = infoDict['sutuserinfo'][0] - # user serial always an integer, see: http://developer.android.com/reference/android/os/UserManager.html#getSerialNumberForUser%28android.os.UserHandle%29 - m = re.match('User Serial:([0-9]+)', userSerialString) - if m: - self._userSerial = m.group(1) - else: - self._userSerial = None + userSerialString = infoDict['sutuserinfo'][0] + # user serial always an integer, see: + # http://developer.android.com/reference/android/os/UserManager.html#getSerialNumberForUser%28android.os.UserHandle%29 + m = re.match('User Serial:([0-9]+)', userSerialString) + if m: + self._userSerial = m.group(1) + else: + self._userSerial = None else: self._userSerial = None if self._userSerial is not None: - return [ "--user", self._userSerial ] + return ["--user", self._userSerial] return [] def getTopActivity(self): - return self._runCmds([{ 'cmd': "activity" }]).strip() + return self._runCmds([{'cmd': "activity"}]).strip() def getAppRoot(self, packageName): - return self._runCmds([{ 'cmd': 'getapproot %s' % packageName }]).strip() + return self._runCmds([{'cmd': 'getapproot %s' % packageName}]).strip() + def DroidConnectByHWID(hwid, timeout=30, **kwargs): - """Try to connect to the given device by waiting for it to show up using mDNS with the given timeout.""" + """Try to connect to the given device by waiting for it to show up using + mDNS with the given timeout.""" zc = Zeroconf(moznetwork.get_ip()) evt = threading.Event() listener = ZeroconfListener(hwid, evt) sb = ServiceBrowser(zc, "_sutagent._tcp.local.", listener) foundIP = None if evt.wait(timeout): # we found the hwid
--- a/testing/mozbase/mozdevice/mozdevice/sutini.py +++ b/testing/mozbase/mozdevice/mozdevice/sutini.py @@ -18,16 +18,17 @@ SCHEMA = {'Registration Server': (('IPAd ('PORT', '28001'), ('HARDWARE', ''), ('POOL', '')), 'Network Settings': (('SSID', ''), ('AUTH', ''), ('ENCR', ''), ('EAP', ''))} + def get_cfg(d, ini_path): cfg = ConfigParser.RawConfigParser() try: cfg.readfp(StringIO.StringIO(d.pullFile(ini_path)), 'SUTAgent.ini') except DMError: # assume this is due to a missing file... pass return cfg
--- a/testing/mozbase/mozdevice/mozdevice/version_codes.py +++ b/testing/mozbase/mozdevice/mozdevice/version_codes.py @@ -12,19 +12,19 @@ See http://developer.android.com/referen # not yet turned into an official release. CUR_DEVELOPMENT = 10000 # October 2008: The original, first, version of Android BASE = 1 # February 2009: First Android update, officially called 1.1 BASE_1_1 = 2 # May 2009: Android 1.5 -CUPCAKE = 3 +CUPCAKE = 3 # September 2009: Android 1.6 -DONUT = 4 +DONUT = 4 # November 2009: Android 2.0 ECLAIR = 5 # December 2009: Android 2.0.1 ECLAIR_0_1 = 6 # January 2010: Android 2.1 ECLAIR_MR1 = 7 # June 2010: Android 2.2 FROYO = 8
--- a/testing/mozbase/mozdevice/setup.py +++ b/testing/mozbase/mozdevice/setup.py @@ -6,23 +6,23 @@ from setuptools import setup PACKAGE_NAME = 'mozdevice' PACKAGE_VERSION = '0.48' deps = ['mozfile >= 1.0', 'mozlog >= 3.0', 'moznetwork >= 0.24', 'mozprocess >= 0.19', - ] + ] setup(name=PACKAGE_NAME, version=PACKAGE_VERSION, description="Mozilla-authored device management", long_description="see http://mozbase.readthedocs.org/", - classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='', author='Mozilla Automation and Testing Team', author_email='tools@lists.mozilla.org', url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', license='MPL', packages=['mozdevice'], include_package_data=True, zip_safe=False,
--- a/testing/mozbase/mozdevice/sut_tests/dmunit.py +++ b/testing/mozbase/mozdevice/sut_tests/dmunit.py @@ -9,16 +9,17 @@ import unittest from mozdevice import devicemanager from mozdevice import devicemanagerSUT ip = '' port = 0 heartbeat_port = 0 log_level = logging.ERROR + class DeviceManagerTestCase(unittest.TestCase): """DeviceManager tests should subclass this. """ """Set to False in your derived class if this test should not be run on the Python agent. """ runs_on_test_device = True @@ -43,12 +44,12 @@ class DeviceManagerTestLoader(unittest.T """Loads tests from modules unless the SUT is a test device and the test case has runs_on_test_device set to False """ tests = [] module = __import__(module_name) for name in dir(module): obj = getattr(module, name) if (isinstance(obj, (type, types.ClassType)) and - issubclass(obj, unittest.TestCase)) and \ - (not self.isTestDevice or obj.runs_on_test_device): + issubclass(obj, unittest.TestCase)) and \ + (not self.isTestDevice or obj.runs_on_test_device): tests.append(self.loadTestsFromTestCase(obj)) return self.suiteClass(tests)
--- a/testing/mozbase/mozdevice/sut_tests/runtests.py +++ b/testing/mozbase/mozdevice/sut_tests/runtests.py @@ -41,17 +41,17 @@ def main(ip, port, heartbeat_port, scrip testLoader = dmunit.DeviceManagerTestLoader(isTestDevice) for s in scripts: suite.addTest(testLoader.loadTestsFromModuleName(s)) unittest.TextTestRunner(verbosity=2).run(suite) genfiles.clean_test_files() -if __name__ == "__main__": +if __name__ == "__main__": default_ip = '127.0.0.1' default_port = 20701 env_ip, _, env_port = os.getenv('TEST_DEVICE', '').partition(':') if env_port: try: env_port = int(env_port)
--- a/testing/mozbase/mozdevice/sut_tests/test_datachannel.py +++ b/testing/mozbase/mozdevice/sut_tests/test_datachannel.py @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import re import socket from time import strptime from dmunit import DeviceManagerTestCase, heartbeat_port + class DataChannelTestCase(DeviceManagerTestCase): runs_on_test_device = False def runTest(self): """This tests the heartbeat and the data channel. """ ip = self.dm.host @@ -31,22 +32,22 @@ class DataChannelTestCase(DeviceManagerT data = self._datasock.recv(1024) print data self.assertNotEqual(len(data), 0) # Check for the header if not capturedHeader: m = re.match(r"(.*?) trace output", data) self.assertNotEqual(m, None, - 'trace output line does not match. The line: ' + str(data)) + 'trace output line does not match. The line: ' + str(data)) capturedHeader = True # Check for standard heartbeat messsage m = re.match(r"(.*?) Thump thump - (.*)", data) - if m == None: + if m is None: # This isn't an error, it usually means we've obtained some # unexpected data from the device continue # Ensure it matches our format mHeartbeatTime = m.group(1) mHeartbeatTime = strptime(mHeartbeatTime, "%Y%m%d-%H:%M:%S") numbeats = numbeats + 1
--- a/testing/mozbase/mozdevice/sut_tests/test_exec.py +++ b/testing/mozbase/mozdevice/sut_tests/test_exec.py @@ -2,16 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import posixpath from StringIO import StringIO from dmunit import DeviceManagerTestCase + class ExecTestCase(DeviceManagerTestCase): def runTest(self): """Simple exec test, does not use env vars.""" out = StringIO() filename = posixpath.join(self.dm.deviceRoot, 'test_exec_file') # Make sure the file was not already there self.dm.removeFile(filename)
--- a/testing/mozbase/mozdevice/sut_tests/test_exec_env.py +++ b/testing/mozbase/mozdevice/sut_tests/test_exec_env.py @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os import posixpath from StringIO import StringIO from dmunit import DeviceManagerTestCase + class ExecEnvTestCase(DeviceManagerTestCase): def runTest(self): """Exec test with env vars.""" # Push the file localfile = os.path.join('test-files', 'test_script.sh') remotefile = posixpath.join(self.dm.deviceRoot, 'test_script.sh') self.dm.pushFile(localfile, remotefile)
--- a/testing/mozbase/mozdevice/sut_tests/test_fileExists.py +++ b/testing/mozbase/mozdevice/sut_tests/test_fileExists.py @@ -2,16 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import tempfile import posixpath from dmunit import DeviceManagerTestCase + class FileExistsTestCase(DeviceManagerTestCase): """This tests the "fileExists" command. """ def testOnRoot(self): self.assertTrue(self.dm.fileExists('/')) def testOnNonexistent(self): @@ -29,9 +30,8 @@ class FileExistsTestCase(DeviceManagerTe remote_path = posixpath.join(self.dm.deviceRoot, 'testDir') remote_path_file = posixpath.join(remote_path, 'testFile') self.assertFalse(self.dm.fileExists(remote_path)) with tempfile.NamedTemporaryFile() as f: self.dm.pushFile(f.name, remote_path_file) self.assertTrue(self.dm.fileExists(remote_path)) self.dm.removeFile(remote_path_file) self.dm.removeDir(remote_path) -
--- a/testing/mozbase/mozdevice/sut_tests/test_getdir.py +++ b/testing/mozbase/mozdevice/sut_tests/test_getdir.py @@ -5,16 +5,17 @@ import os import posixpath import shutil import tempfile from mozdevice.devicemanager import DMError from dmunit import DeviceManagerTestCase + class GetDirectoryTestCase(DeviceManagerTestCase): def _setUp(self): self.localsrcdir = tempfile.mkdtemp() os.makedirs(os.path.join(self.localsrcdir, 'push1', 'sub.1', 'sub.2')) path = os.path.join(self.localsrcdir, 'push1', 'sub.1', 'sub.2', 'testfile') file(path, 'w').close() @@ -41,10 +42,10 @@ class GetDirectoryTestCase(DeviceManager self.dm.getDirectory(posixpath.join(testroot, 'push1'), os.path.join(self.localdestdir, 'push1')) self.assertTrue(os.path.exists( os.path.join(self.localdestdir, 'push1', 'sub.1', 'sub.2', 'testfile'))) self.assertTrue(os.path.exists( os.path.join(self.localdestdir, 'push1', 'emptysub'))) self.assertRaises(DMError, self.dm.getDirectory, - '/dummy', os.path.join(self.localdestdir, '/none')) + '/dummy', os.path.join(self.localdestdir, '/none')) self.assertFalse(os.path.exists(self.localdestdir + '/none'))
--- a/testing/mozbase/mozdevice/sut_tests/test_info.py +++ b/testing/mozbase/mozdevice/sut_tests/test_info.py @@ -1,14 +1,15 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from dmunit import DeviceManagerTestCase + class InfoTestCase(DeviceManagerTestCase): runs_on_test_device = False def runTest(self): """This tests the "info" command. """ cmds = ('os', 'id', 'systime', 'uptime', 'screen', 'memory', 'power')
--- a/testing/mozbase/mozdevice/sut_tests/test_prompt.py +++ b/testing/mozbase/mozdevice/sut_tests/test_prompt.py @@ -2,16 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import re import socket from dmunit import DeviceManagerTestCase + class PromptTestCase(DeviceManagerTestCase): def tearDown(self): if self.sock: self.sock.close() def runTest(self): """This tests getting a prompt from the device.
--- a/testing/mozbase/mozdevice/sut_tests/test_ps.py +++ b/testing/mozbase/mozdevice/sut_tests/test_ps.py @@ -1,14 +1,15 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from dmunit import DeviceManagerTestCase + class ProcessListTestCase(DeviceManagerTestCase): def runTest(self): """This tests getting a process list from the device. """ proclist = self.dm.getProcessList() # This returns a process list of the form: @@ -19,9 +20,8 @@ class ProcessListTestCase(DeviceManagerT self.assertNotEqual(len(proclist), 0) for item in proclist: self.assertIsInstance(item[0], int) self.assertIsInstance(item[1], str) self.assertGreater(len(item[1]), 0) if len(item) > 2: self.assertIsInstance(item[2], int) -
--- a/testing/mozbase/mozdevice/sut_tests/test_pull.py +++ b/testing/mozbase/mozdevice/sut_tests/test_pull.py @@ -4,16 +4,17 @@ import hashlib import os import posixpath from dmunit import DeviceManagerTestCase from mozdevice.devicemanager import DMError + class PullTestCase(DeviceManagerTestCase): def runTest(self): """Tests the "pull" command with a binary file. """ orig = hashlib.md5() new = hashlib.md5() local_test_file = os.path.join('test-files', 'mybinary.zip')
--- a/testing/mozbase/mozdevice/sut_tests/test_push1.py +++ b/testing/mozbase/mozdevice/sut_tests/test_push1.py @@ -2,16 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os import posixpath from dmunit import DeviceManagerTestCase + class Push1TestCase(DeviceManagerTestCase): def runTest(self): """This tests copying a directory structure to the device. """ dvroot = self.dm.deviceRoot dvpath = posixpath.join(dvroot, 'infratest') self.dm.removeDir(dvpath)
--- a/testing/mozbase/mozdevice/sut_tests/test_push2.py +++ b/testing/mozbase/mozdevice/sut_tests/test_push2.py @@ -2,16 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os import posixpath from dmunit import DeviceManagerTestCase + class Push2TestCase(DeviceManagerTestCase): def runTest(self): """This tests copying a directory structure with files to the device. """ testroot = posixpath.join(self.dm.deviceRoot, 'infratest') self.dm.removeDir(testroot) self.dm.mkDir(testroot)
--- a/testing/mozbase/mozdevice/sut_tests/test_pushbinary.py +++ b/testing/mozbase/mozdevice/sut_tests/test_pushbinary.py @@ -2,16 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os import posixpath from dmunit import DeviceManagerTestCase + class PushBinaryTestCase(DeviceManagerTestCase): def runTest(self): """This tests copying a binary file. """ testroot = self.dm.deviceRoot self.dm.removeFile(posixpath.join(testroot, 'mybinary.zip')) self.dm.pushFile(os.path.join('test-files', 'mybinary.zip'),
--- a/testing/mozbase/mozdevice/sut_tests/test_pushsmalltext.py +++ b/testing/mozbase/mozdevice/sut_tests/test_pushsmalltext.py @@ -2,16 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os import posixpath from dmunit import DeviceManagerTestCase + class PushSmallTextTestCase(DeviceManagerTestCase): def runTest(self): """This tests copying a small text file. """ testroot = self.dm.deviceRoot self.dm.removeFile(posixpath.join(testroot, 'smalltext.txt')) self.dm.pushFile(os.path.join('test-files', 'smalltext.txt'),
--- a/testing/mozbase/mozdevice/tests/droidsut_launch.py +++ b/testing/mozbase/mozdevice/tests/droidsut_launch.py @@ -1,35 +1,36 @@ from sut import MockAgent import mozdevice import logging import unittest + class LaunchTest(unittest.TestCase): def test_nouserserial(self): - a = MockAgent(self, commands = [("ps", - "10029 549 com.android.launcher\n" - "10066 1198 com.twitter.android"), - ("info sutuserinfo", ""), - ("exec am start -W -n " - "org.mozilla.fennec/org.mozilla.gecko.BrowserApp -a " - "android.intent.action.VIEW", - "OK\nreturn code [0]")]) + a = MockAgent(self, commands=[("ps", + "10029 549 com.android.launcher\n" + "10066 1198 com.twitter.android"), + ("info sutuserinfo", ""), + ("exec am start -W -n " + "org.mozilla.fennec/org.mozilla.gecko.BrowserApp -a " + "android.intent.action.VIEW", + "OK\nreturn code [0]")]) d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) d.launchFennec("org.mozilla.fennec") a.wait() def test_userserial(self): - a = MockAgent(self, commands = [("ps", - "10029 549 com.android.launcher\n" - "10066 1198 com.twitter.android"), - ("info sutuserinfo", "User Serial:0"), - ("exec am start --user 0 -W -n " - "org.mozilla.fennec/org.mozilla.gecko.BrowserApp -a " - "android.intent.action.VIEW", - "OK\nreturn code [0]")]) + a = MockAgent(self, commands=[("ps", + "10029 549 com.android.launcher\n" + "10066 1198 com.twitter.android"), + ("info sutuserinfo", "User Serial:0"), + ("exec am start --user 0 -W -n " + "org.mozilla.fennec/org.mozilla.gecko.BrowserApp -a " + "android.intent.action.VIEW", + "OK\nreturn code [0]")]) d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) d.launchFennec("org.mozilla.fennec") a.wait() if __name__ == '__main__': unittest.main()
--- a/testing/mozbase/mozdevice/tests/sut.py +++ b/testing/mozbase/mozdevice/tests/sut.py @@ -4,22 +4,23 @@ # http://creativecommons.org/publicdomain/zero/1.0/ import datetime import socket import time from threading import Thread + class MockAgent(object): MAX_WAIT_TIME_SECONDS = 10 SOCKET_TIMEOUT_SECONDS = 5 - def __init__(self, tester, start_commands = None, commands = []): + def __init__(self, tester, start_commands=None, commands=[]): if start_commands: self.commands = start_commands else: self.commands = [("ver", "SUTAgentAndroid Version 1.14")] self.commands = self.commands + commands self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._sock.bind(("127.0.0.1", 0)) @@ -55,21 +56,21 @@ class MockAgent(object): data += conn.recv(1024) except socket.timeout: # We handle timeouts in the main loop. pass self.tester.assertEqual(data.strip(), command) # send response and prompt separately to test for bug 789496 # FIXME: Improve the mock agent, since overloading the meaning # of 'response' is getting confusing. - if response is None: # code for "shut down" + if response is None: # code for "shut down" conn.shutdown(socket.SHUT_RDWR) conn.close() conn = None - elif type(response) is int: # code for "time out" + elif type(response) is int: # code for "time out" max_timeout = 15.0 timeout = 0.0 interval = 0.1 while not self.should_stop and timeout < max_timeout: time.sleep(interval) timeout += interval if timeout >= max_timeout: raise Exception("Maximum timeout reached! This should not "
--- a/testing/mozbase/mozdevice/tests/sut_app.py +++ b/testing/mozbase/mozdevice/tests/sut_app.py @@ -1,9 +1,9 @@ -#/usr/bin/env python +#!/usr/bin/env python import mozdevice import logging import unittest from sut import MockAgent class TestApp(unittest.TestCase):
--- a/testing/mozbase/mozdevice/tests/sut_basic.py +++ b/testing/mozbase/mozdevice/tests/sut_basic.py @@ -1,13 +1,14 @@ from sut import MockAgent import mozdevice import logging import unittest + class BasicTest(unittest.TestCase): def test_init(self): """Tests DeviceManager initialization.""" a = MockAgent(self) mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) # all testing done in device's constructor @@ -19,47 +20,47 @@ class BasicTest(unittest.TestCase): self.assertRaises(mozdevice.DMError, lambda: mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG)) a.wait() def test_timeout_normal(self): """Tests DeviceManager timeout, normal case.""" - a = MockAgent(self, commands = [("isdir /mnt/sdcard/tests", "TRUE"), - ("cd /mnt/sdcard/tests", ""), - ("ls", "test.txt"), - ("rm /mnt/sdcard/tests/test.txt", - "Removed the file")]) + a = MockAgent(self, commands=[("isdir /mnt/sdcard/tests", "TRUE"), + ("cd /mnt/sdcard/tests", ""), + ("ls", "test.txt"), + ("rm /mnt/sdcard/tests/test.txt", + "Removed the file")]) d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) ret = d.removeFile('/mnt/sdcard/tests/test.txt') - self.assertEqual(ret, None) # if we didn't throw an exception, we're ok + self.assertEqual(ret, None) # if we didn't throw an exception, we're ok a.wait() def test_timeout_timeout(self): """Tests DeviceManager timeout, timeout case.""" - a = MockAgent(self, commands = [("isdir /mnt/sdcard/tests", "TRUE"), - ("cd /mnt/sdcard/tests", ""), - ("ls", "test.txt"), - ("rm /mnt/sdcard/tests/test.txt", 0)]) + a = MockAgent(self, commands=[("isdir /mnt/sdcard/tests", "TRUE"), + ("cd /mnt/sdcard/tests", ""), + ("ls", "test.txt"), + ("rm /mnt/sdcard/tests/test.txt", 0)]) d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) d.default_timeout = 1 exceptionThrown = False try: d.removeFile('/mnt/sdcard/tests/test.txt') except mozdevice.DMError: exceptionThrown = True self.assertEqual(exceptionThrown, True) a.should_stop = True a.wait() def test_shell(self): """Tests shell command""" - for cmd in [ ("exec foobar", False), ("execsu foobar", True) ]: - for retcode in [ 1, 2 ]: + for cmd in [("exec foobar", False), ("execsu foobar", True)]: + for retcode in [1, 2]: a = MockAgent(self, commands=[(cmd[0], "\nreturn code [%s]" % retcode)]) d = mozdevice.DroidSUT("127.0.0.1", port=a.port) exceptionThrown = False try: d.shellCheckOutput(["foobar"], root=cmd[1]) except mozdevice.DMError: exceptionThrown = True
--- a/testing/mozbase/mozdevice/tests/sut_chmod.py +++ b/testing/mozbase/mozdevice/tests/sut_chmod.py @@ -1,21 +1,22 @@ -#/usr/bin/env python +#!/usr/bin/env python import mozdevice import logging import unittest from sut import MockAgent class TestChmod(unittest.TestCase): def test_chmod(self): - command = [('chmod /mnt/sdcard/test', 'Changing permissions for /storage/emulated/legacy/Test\n' - ' <empty>\n' - 'chmod /storage/emulated/legacy/Test ok\n')] + command = [('chmod /mnt/sdcard/test', + 'Changing permissions for /storage/emulated/legacy/Test\n' + ' <empty>\n' + 'chmod /storage/emulated/legacy/Test ok\n')] m = MockAgent(self, commands=command) d = mozdevice.DroidSUT('127.0.0.1', port=m.port, logLevel=logging.DEBUG) self.assertEqual(None, d.chmodDir('/mnt/sdcard/test')) if __name__ == '__main__': unittest.main()
--- a/testing/mozbase/mozdevice/tests/sut_copytree.py +++ b/testing/mozbase/mozdevice/tests/sut_copytree.py @@ -5,61 +5,63 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import mozdevice import logging import unittest from sut import MockAgent + class CopyTreeTest(unittest.TestCase): + def test_copyFile(self): commands = [('dd if=/mnt/sdcard/tests/test.txt of=/mnt/sdcard/tests/test2.txt', ''), ('isdir /mnt/sdcard/tests', 'TRUE'), ('cd /mnt/sdcard/tests', ''), ('ls', 'test.txt\ntest2.txt')] m = MockAgent(self, commands=commands) d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) self.assertEqual(None, d.copyTree('/mnt/sdcard/tests/test.txt', - '/mnt/sdcard/tests/test2.txt')) + '/mnt/sdcard/tests/test2.txt')) expected = (commands[3][1].strip()).split('\n') self.assertEqual(expected, d.listFiles('/mnt/sdcard/tests')) def test_copyDir(self): commands = [('dd if=/mnt/sdcard/tests/foo of=/mnt/sdcard/tests/bar', ''), ('isdir /mnt/sdcard/tests', 'TRUE'), ('cd /mnt/sdcard/tests', ''), ('ls', 'foo\nbar')] m = MockAgent(self, commands=commands) d = mozdevice.DroidSUT("127.0.0.1", port=m.port, - logLevel=logging.DEBUG) + logLevel=logging.DEBUG) self.assertEqual(None, d.copyTree('/mnt/sdcard/tests/foo', - '/mnt/sdcard/tests/bar')) + '/mnt/sdcard/tests/bar')) expected = (commands[3][1].strip()).split('\n') self.assertEqual(expected, d.listFiles('/mnt/sdcard/tests')) def test_copyNonEmptyDir(self): commands = [('isdir /mnt/sdcard/tests/foo/bar', 'TRUE'), ('dd if=/mnt/sdcard/tests/foo of=/mnt/sdcard/tests/foo2', ''), ('isdir /mnt/sdcard/tests', 'TRUE'), ('cd /mnt/sdcard/tests', ''), ('ls', 'foo\nfoo2'), ('isdir /mnt/sdcard/tests/foo2', 'TRUE'), ('cd /mnt/sdcard/tests/foo2', ''), ('ls', 'bar')] m = MockAgent(self, commands=commands) d = mozdevice.DroidSUT("127.0.0.1", port=m.port, - logLevel=logging.DEBUG) + logLevel=logging.DEBUG) self.assertTrue(d.dirExists('/mnt/sdcard/tests/foo/bar')) self.assertEqual(None, d.copyTree('/mnt/sdcard/tests/foo', - '/mnt/sdcard/tests/foo2')) + '/mnt/sdcard/tests/foo2')) expected = (commands[4][1].strip()).split('\n') self.assertEqual(expected, d.listFiles('/mnt/sdcard/tests')) self.assertTrue(d.fileExists('/mnt/sdcard/tests/foo2/bar')) if __name__ == "__main__": unittest.main()
--- a/testing/mozbase/mozdevice/tests/sut_fileExists.py +++ b/testing/mozbase/mozdevice/tests/sut_fileExists.py @@ -1,12 +1,13 @@ from sut import MockAgent import mozdevice import unittest + class FileExistsTest(unittest.TestCase): commands = [('isdir /', 'TRUE'), ('cd /', ''), ('ls', 'init')] def test_onRoot(self): root_commands = [('isdir /', 'TRUE')] @@ -21,9 +22,8 @@ class FileExistsTest(unittest.TestCase): def test_onRegularFile(self): a = MockAgent(self, commands=self.commands) d = mozdevice.DroidSUT("127.0.0.1", port=a.port) self.assertTrue(d.fileExists('/init')) if __name__ == '__main__': unittest.main() -
--- a/testing/mozbase/mozdevice/tests/sut_info.py +++ b/testing/mozbase/mozdevice/tests/sut_info.py @@ -1,9 +1,9 @@ -#/usr/bin/env python +#!/usr/bin/env python import mozdevice import logging import re import unittest from sut import MockAgent class TestGetInfo(unittest.TestCase):
--- a/testing/mozbase/mozdevice/tests/sut_ip.py +++ b/testing/mozbase/mozdevice/tests/sut_ip.py @@ -1,9 +1,9 @@ -#/usr/bin/env python +#!/usr/bin/env python import mozdevice import logging import unittest from sut import MockAgent class TestGetIP(unittest.TestCase): """ class to test IP methods """
--- a/testing/mozbase/mozdevice/tests/sut_list.py +++ b/testing/mozbase/mozdevice/tests/sut_list.py @@ -1,9 +1,9 @@ -#/usr/bin/env python +#!/usr/bin/env python import mozdevice import logging import unittest from sut import MockAgent class TestListFiles(unittest.TestCase): commands = [("isdir /mnt/sdcard", "TRUE"),
--- a/testing/mozbase/mozdevice/tests/sut_logcat.py +++ b/testing/mozbase/mozdevice/tests/sut_logcat.py @@ -6,37 +6,38 @@ import unittest from sut import MockAgent class TestLogCat(unittest.TestCase): """ Class to test methods associated with logcat """ def test_getLogcat(self): - logcat_output = ("07-17 00:51:10.377 I/SUTAgentAndroid( 2933): onCreate\r\n" - "07-17 00:51:10.457 D/dalvikvm( 2933): GC_CONCURRENT freed 351K, 17% free 2523K/3008K, paused 5ms+2ms, total 38ms\r\n" - "07-17 00:51:10.497 I/SUTAgentAndroid( 2933): Caught exception creating file in /data/local/tmp: open failed: EACCES (Permission denied)\r\n" - "07-17 00:51:10.507 E/SUTAgentAndroid( 2933): ERROR: Cannot access world writeable test root\r\n" - "07-17 00:51:10.547 D/GeckoHealthRec( 3253): Initializing profile cache.\r\n" - "07-17 00:51:10.607 D/GeckoHealthRec( 3253): Looking for /data/data/org.mozilla.fennec/files/mozilla/c09kfhne.default/times.json\r\n" - "07-17 00:51:10.637 D/GeckoHealthRec( 3253): Using times.json for profile creation time.\r\n" - "07-17 00:51:10.707 D/GeckoHealthRec( 3253): Incorporating environment: times.json profile creation = 1374026758604\r\n" - "07-17 00:51:10.507 D/GeckoHealthRec( 3253): Requested prefs.\r\n" - "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): \r\n" - "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Private Dirty Memory 3176 kb\r\n" - "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Proportional Set Size Memory 5679 kb\r\n" - "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Shared Dirty Memory 9216 kb\r\n" - "07-17 06:55:21.627 I/SUTAgentAndroid( 3876): 127.0.0.1 : execsu /system/bin/logcat -v time -d dalvikvm:I " - "ConnectivityService:S WifiMonitor:S WifiStateTracker:S wpa_supplicant:S NetworkStateTracker:S\r\n" - "07-17 06:55:21.827 I/dalvikvm-heap( 3876): Grow heap (frag case) to 3.019MB for 102496-byte allocation\r\n" - "return code [0]") + logcat_output = ( + "07-17 00:51:10.377 I/SUTAgentAndroid( 2933): onCreate\r\n" + "07-17 00:51:10.457 D/dalvikvm( 2933): GC_CONCURRENT freed 351K, 17% free 2523K/3008K, paused 5ms+2ms, total 38ms\r\n" # noqa + "07-17 00:51:10.497 I/SUTAgentAndroid( 2933): Caught exception creating file in /data/local/tmp: open failed: EACCES (Permission denied)\r\n" # noqa + "07-17 00:51:10.507 E/SUTAgentAndroid( 2933): ERROR: Cannot access world writeable test root\r\n" # noqa + "07-17 00:51:10.547 D/GeckoHealthRec( 3253): Initializing profile cache.\r\n" + "07-17 00:51:10.607 D/GeckoHealthRec( 3253): Looking for /data/data/org.mozilla.fennec/files/mozilla/c09kfhne.default/times.json\r\n" # noqa + "07-17 00:51:10.637 D/GeckoHealthRec( 3253): Using times.json for profile creation time.\r\n" # noqa + "07-17 00:51:10.707 D/GeckoHealthRec( 3253): Incorporating environment: times.json profile creation = 1374026758604\r\n" # noqa + "07-17 00:51:10.507 D/GeckoHealthRec( 3253): Requested prefs.\r\n" + "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): \r\n" + "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Private Dirty Memory 3176 kb\r\n" # noqa + "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Proportional Set Size Memory 5679 kb\r\n" # noqa + "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Shared Dirty Memory 9216 kb\r\n" # noqa + "07-17 06:55:21.627 I/SUTAgentAndroid( 3876): 127.0.0.1 : execsu /system/bin/logcat -v time -d dalvikvm:I " # noqa + "ConnectivityService:S WifiMonitor:S WifiStateTracker:S wpa_supplicant:S NetworkStateTracker:S\r\n" # noqa + "07-17 06:55:21.827 I/dalvikvm-heap( 3876): Grow heap (frag case) to 3.019MB for 102496-byte allocation\r\n" # noqa + "return code [0]") inp = ("execsu /system/bin/logcat -v time -d " - "dalvikvm:I ConnectivityService:S WifiMonitor:S " - "WifiStateTracker:S wpa_supplicant:S NetworkStateTracker:S") + "dalvikvm:I ConnectivityService:S WifiMonitor:S " + "WifiStateTracker:S wpa_supplicant:S NetworkStateTracker:S") commands = [(inp, logcat_output)] m = MockAgent(self, commands=commands) d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) self.assertEqual(logcat_output[:-17].replace('\r\n', '\n').splitlines(True), d.getLogcat()) def test_recordLogcat(self):
--- a/testing/mozbase/mozdevice/tests/sut_mkdir.py +++ b/testing/mozbase/mozdevice/tests/sut_mkdir.py @@ -1,16 +1,17 @@ # Any copyright is dedicated to the Public Domain. # http://creativecommons.org/publicdomain/zero/1.0/ import mozdevice import logging import unittest from sut import MockAgent + class MkDirsTest(unittest.TestCase): def test_mkdirs(self): subTests = [{'cmds': [('isdir /mnt/sdcard/baz/boop', 'FALSE'), ('info os', 'android'), ('isdir /mnt', 'TRUE'), ('isdir /mnt/sdcard', 'TRUE'), ('isdir /mnt/sdcard/baz', 'FALSE'), @@ -21,19 +22,20 @@ class MkDirsTest(unittest.TestCase): '/mnt/sdcard/baz/boop successfully created')], 'expectException': False}, {'cmds': [('isdir /mnt/sdcard/baz/boop', 'FALSE'), ('info os', 'android'), ('isdir /mnt', 'TRUE'), ('isdir /mnt/sdcard', 'TRUE'), ('isdir /mnt/sdcard/baz', 'FALSE'), ('mkdr /mnt/sdcard/baz', - '##AGENT-WARNING## Could not create the directory /mnt/sdcard/baz')], + "##AGENT-WARNING## " + "Could not create the directory /mnt/sdcard/baz")], 'expectException': True}, - ] + ] for subTest in subTests: a = MockAgent(self, commands=subTest['cmds']) exceptionThrown = False try: d = mozdevice.DroidSUT('127.0.0.1', port=a.port, logLevel=logging.DEBUG) d.mkDirs('/mnt/sdcard/baz/boop/bip')
--- a/testing/mozbase/mozdevice/tests/sut_movetree.py +++ b/testing/mozbase/mozdevice/tests/sut_movetree.py @@ -5,59 +5,61 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import mozdevice import logging import unittest from sut import MockAgent + class MoveTreeTest(unittest.TestCase): + def test_moveFile(self): commands = [('mv /mnt/sdcard/tests/test.txt /mnt/sdcard/tests/test1.txt', ''), ('isdir /mnt/sdcard/tests', 'TRUE'), ('cd /mnt/sdcard/tests', ''), ('ls', 'test1.txt'), ('isdir /mnt/sdcard/tests', 'TRUE'), ('cd /mnt/sdcard/tests', ''), ('ls', 'test1.txt')] m = MockAgent(self, commands=commands) d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) self.assertEqual(None, d.moveTree('/mnt/sdcard/tests/test.txt', - '/mnt/sdcard/tests/test1.txt')) + '/mnt/sdcard/tests/test1.txt')) self.assertFalse(d.fileExists('/mnt/sdcard/tests/test.txt')) self.assertTrue(d.fileExists('/mnt/sdcard/tests/test1.txt')) def test_moveDir(self): commands = [("mv /mnt/sdcard/tests/foo /mnt/sdcard/tests/bar", ""), ('isdir /mnt/sdcard/tests', 'TRUE'), ('cd /mnt/sdcard/tests', ''), ('ls', 'bar')] m = MockAgent(self, commands=commands) d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) self.assertEqual(None, d.moveTree('/mnt/sdcard/tests/foo', - '/mnt/sdcard/tests/bar')) + '/mnt/sdcard/tests/bar')) self.assertTrue(d.fileExists('/mnt/sdcard/tests/bar')) def test_moveNonEmptyDir(self): commands = [('isdir /mnt/sdcard/tests/foo/bar', 'TRUE'), ('mv /mnt/sdcard/tests/foo /mnt/sdcard/tests/foo2', ''), ('isdir /mnt/sdcard/tests', 'TRUE'), ('cd /mnt/sdcard/tests', ''), ('ls', 'foo2'), ('isdir /mnt/sdcard/tests/foo2', 'TRUE'), ('cd /mnt/sdcard/tests/foo2', ''), ('ls', 'bar')] m = MockAgent(self, commands=commands) d = mozdevice.DroidSUT("127.0.0.1", port=m.port, - logLevel=logging.DEBUG) + logLevel=logging.DEBUG) self.assertTrue(d.dirExists('/mnt/sdcard/tests/foo/bar')) self.assertEqual(None, d.moveTree('/mnt/sdcard/tests/foo', - '/mnt/sdcard/tests/foo2')) + '/mnt/sdcard/tests/foo2')) self.assertTrue(d.fileExists('/mnt/sdcard/tests/foo2')) self.assertTrue(d.fileExists('/mnt/sdcard/tests/foo2/bar')) if __name__ == "__main__": unittest.main()
--- a/testing/mozbase/mozdevice/tests/sut_ps.py +++ b/testing/mozbase/mozdevice/tests/sut_ps.py @@ -1,12 +1,13 @@ from sut import MockAgent import mozdevice import unittest + class PsTest(unittest.TestCase): pscommands = [('ps', "10029 549 com.android.launcher\n" "10066 1198 com.twitter.android")] bad_pscommands = [('ps', "abcdef 549 com.android.launcher\n"
--- a/testing/mozbase/mozdevice/tests/sut_pull.py +++ b/testing/mozbase/mozdevice/tests/sut_pull.py @@ -1,48 +1,47 @@ from sut import MockAgent import mozdevice import logging import unittest + class PullTest(unittest.TestCase): def test_pull_success(self): - for count in [ 1, 4, 1024, 2048 ]: + for count in [1, 4, 1024, 2048]: cheeseburgers = "" for i in range(count): cheeseburgers += "cheeseburgers" # pull file is kind of gross, make sure we can still execute commands after it's done remoteName = "/mnt/sdcard/cheeseburgers" - a = MockAgent(self, commands = [("pull %s" % remoteName, - "%s,%s\n%s" % (remoteName, - len(cheeseburgers), - cheeseburgers)), - ("isdir /mnt/sdcard", "TRUE")]) + a = MockAgent(self, commands=[("pull %s" % remoteName, + "%s,%s\n%s" % (remoteName, + len(cheeseburgers), + cheeseburgers)), + ("isdir /mnt/sdcard", "TRUE")]) d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) pulledData = d.pullFile("/mnt/sdcard/cheeseburgers") self.assertEqual(pulledData, cheeseburgers) d.dirExists('/mnt/sdcard') def test_pull_failure(self): # this test simulates only receiving a few bytes of what we expect # to be larger file remoteName = "/mnt/sdcard/cheeseburgers" - a = MockAgent(self, commands = [("pull %s" % remoteName, - "%s,15\n%s" % (remoteName, - "cheeseburgh"))]) + a = MockAgent(self, commands=[("pull %s" % remoteName, + "%s,15\n%s" % (remoteName, + "cheeseburgh"))]) d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) exceptionThrown = False try: d.pullFile("/mnt/sdcard/cheeseburgers") except mozdevice.DMError: exceptionThrown = True self.assertTrue(exceptionThrown) if __name__ == '__main__': unittest.main() - -
--- a/testing/mozbase/mozdevice/tests/sut_push.py +++ b/testing/mozbase/mozdevice/tests/sut_push.py @@ -2,29 +2,30 @@ from sut import MockAgent import mozfile import mozdevice import logging import unittest import hashlib import tempfile import os + class PushTest(unittest.TestCase): def test_push(self): pushfile = "1234ABCD" mdsum = hashlib.md5() mdsum.update(pushfile) expectedResponse = mdsum.hexdigest() # (good response, no exception), (bad response, exception) - for response in [ (expectedResponse, False), ("BADHASH", True) ]: + for response in [(expectedResponse, False), ("BADHASH", True)]: cmd = "push /mnt/sdcard/foobar %s\r\n%s" % (len(pushfile), pushfile) - a = MockAgent(self, commands = [("isdir /mnt/sdcard", "TRUE"), - (cmd, response[0])]) + a = MockAgent(self, commands=[("isdir /mnt/sdcard", "TRUE"), + (cmd, response[0])]) exceptionThrown = False with tempfile.NamedTemporaryFile() as f: try: f.write(pushfile) f.flush() d = mozdevice.DroidSUT("127.0.0.1", port=a.port) d.pushFile(f.name, '/mnt/sdcard/foobar') except mozdevice.DMError: @@ -41,39 +42,39 @@ class PushTest(unittest.TestCase): tempdir = tempfile.mkdtemp() self.addCleanup(mozfile.remove, tempdir) complex_path = os.path.join(tempdir, "baz") os.mkdir(complex_path) f = tempfile.NamedTemporaryFile(dir=complex_path) f.write(pushfile) f.flush() - subTests = [ { 'cmds': [ ("isdir /mnt/sdcard/baz", "TRUE"), - ("push /mnt/sdcard/baz/%s %s\r\n%s" % - (os.path.basename(f.name), len(pushfile), - pushfile), - expectedFileResponse) ], - 'expectException': False }, - { 'cmds': [ ("isdir /mnt/sdcard/baz", "TRUE"), - ("push /mnt/sdcard/baz/%s %s\r\n%s" % - (os.path.basename(f.name), len(pushfile), - pushfile), - "BADHASH") ], - 'expectException': True }, - { 'cmds': [ ("isdir /mnt/sdcard/baz", "FALSE"), - ('info os', 'android'), - ("isdir /mnt", "FALSE"), - ("mkdr /mnt", - "##AGENT-WARNING## Could not create the directory /mnt") ], - 'expectException': True }, + subTests = [{'cmds': [("isdir /mnt/sdcard/baz", "TRUE"), + ("push /mnt/sdcard/baz/%s %s\r\n%s" % + (os.path.basename(f.name), len(pushfile), + pushfile), + expectedFileResponse)], + 'expectException': False}, + {'cmds': [("isdir /mnt/sdcard/baz", "TRUE"), + ("push /mnt/sdcard/baz/%s %s\r\n%s" % + (os.path.basename(f.name), len(pushfile), + pushfile), + "BADHASH")], + 'expectException': True}, + {'cmds': [("isdir /mnt/sdcard/baz", "FALSE"), + ('info os', 'android'), + ("isdir /mnt", "FALSE"), + ("mkdr /mnt", + "##AGENT-WARNING## Could not create the directory /mnt")], + 'expectException': True}, - ] + ] for subTest in subTests: - a = MockAgent(self, commands = subTest['cmds']) + a = MockAgent(self, commands=subTest['cmds']) exceptionThrown = False try: d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) d.pushDir(tempdir, "/mnt/sdcard") except mozdevice.DMError: exceptionThrown = True
--- a/testing/mozbase/mozdevice/tests/sut_remove.py +++ b/testing/mozbase/mozdevice/tests/sut_remove.py @@ -1,24 +1,24 @@ -#/usr/bin/env python +#!/usr/bin/env python import mozdevice import logging import unittest from sut import MockAgent class TestRemove(unittest.TestCase): def test_removeDir(self): commands = [("isdir /mnt/sdcard/test", "TRUE"), ("rmdr /mnt/sdcard/test", "Deleting file(s) from " - "/storage/emulated/legacy/Moztest\n" - " <empty>\n" - "Deleting directory " - "/storage/emulated/legacy/Moztest\n")] + "/storage/emulated/legacy/Moztest\n" + " <empty>\n" + "Deleting directory " + "/storage/emulated/legacy/Moztest\n")] m = MockAgent(self, commands=commands) d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) # No error implies we're all good self.assertEqual(None, d.removeDir("/mnt/sdcard/test")) if __name__ == '__main__': unittest.main()
--- a/testing/mozbase/mozdevice/tests/sut_time.py +++ b/testing/mozbase/mozdevice/tests/sut_time.py @@ -1,9 +1,9 @@ -#/usr/bin/env python +#!/usr/bin/env python import mozdevice import logging import unittest from sut import MockAgent class TestGetCurrentTime(unittest.TestCase):
--- a/testing/mozbase/mozfile/mozfile/__init__.py +++ b/testing/mozbase/mozfile/mozfile/__init__.py @@ -1,7 +1,8 @@ +# flake8: noqa # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import from .mozfile import *
--- a/testing/mozbase/mozfile/mozfile/mozfile.py +++ b/testing/mozbase/mozfile/mozfile/mozfile.py @@ -22,17 +22,18 @@ import warnings 'load', 'move', 'remove', 'rmtree', 'tree', 'NamedTemporaryFile', 'TemporaryDirectory'] -### utilities for extracting archives +# utilities for extracting archives + def extract_tarball(src, dest): """extract a .tar file""" import tarfile bundle = tarfile.open(src) namelist = bundle.getnames() @@ -117,17 +118,17 @@ def extract(src, dest=None): if index != -1: root = os.path.join(dest, name[:index]) if root not in top_level_files: top_level_files.append(root) return top_level_files -### utilities for removal of files and directories +# utilities for removal of files and directories def rmtree(dir): """Deprecated wrapper method to remove a directory tree. Ensure to update your code to use mozfile.remove() directly :param dir: directory to be removed """ @@ -156,17 +157,17 @@ def _call_windows_retry(func, args=(), r raise if retry_count == retry_max: raise retry_count += 1 print '%s() failed for "%s". Reason: %s (%s). Retrying...' % \ - (func.__name__, args, e.strerror, e.errno) + (func.__name__, args, e.strerror, e.errno) time.sleep(retry_count * retry_delay) else: # If no exception has been thrown it should be done break def remove(path): """Removes the specified file, link, or directory tree. @@ -256,27 +257,28 @@ def depth(directory): level += 1 if not remainder: break return level # ASCII delimeters ascii_delimeters = { - 'vertical_line' : '|', - 'item_marker' : '+', - 'last_child' : '\\' - } + 'vertical_line': '|', + 'item_marker': '+', + 'last_child': '\\' +} # unicode delimiters unicode_delimeters = { - 'vertical_line' : '│', - 'item_marker' : '├', - 'last_child' : '└' - } + 'vertical_line': '│', + 'item_marker': '├', + 'last_child': '└' +} + def tree(directory, item_marker=unicode_delimeters['item_marker'], vertical_line=unicode_delimeters['vertical_line'], last_child=unicode_delimeters['last_child'], sort_key=lambda x: x.lower()): """ display tree directory structure for `directory` @@ -314,31 +316,31 @@ def tree(directory, indent[-1] = ' ' elif not indent: dirpath_mark = '' else: dirpath_mark = item_marker # append the directory and piece of tree structure # if the top-level entry directory, print as passed - retval.append('%s%s%s'% (''.join(indent[:-1]), - dirpath_mark, - basename if retval else directory)) + retval.append('%s%s%s' % (''.join(indent[:-1]), + dirpath_mark, + basename if retval else directory)) # add the files if filenames: last_file = filenames[-1] retval.extend([('%s%s%s' % (''.join(indent), files_end if filename == last_file else item_marker, filename)) - for index, filename in enumerate(filenames)]) + for index, filename in enumerate(filenames)]) return '\n'.join(retval) -### utilities for temporary resources +# utilities for temporary resources class NamedTemporaryFile(object): """ Like tempfile.NamedTemporaryFile except it works on Windows in the case where you open the created file a second time. This behaves very similarly to tempfile.NamedTemporaryFile but may not behave exactly the same. For example, this function does not @@ -348,16 +350,17 @@ class NamedTemporaryFile(object): with NamedTemporaryFile() as fh: fh.write(b'foobar') print('Filename: %s' % fh.name) see https://bugzilla.mozilla.org/show_bug.cgi?id=821362 """ + def __init__(self, mode='w+b', bufsize=-1, suffix='', prefix='tmp', dir=None, delete=True): import tempfile fd, path = tempfile.mkstemp(suffix, prefix, dir, 't' in mode) os.close(fd) self.file = open(path, mode) @@ -405,31 +408,32 @@ def TemporaryDirectory(): tempdir = tempfile.mkdtemp() try: yield tempdir finally: shutil.rmtree(tempdir) -### utilities dealing with URLs +# utilities dealing with URLs def is_url(thing): """ Return True if thing looks like a URL. """ import urlparse parsed = urlparse.urlparse(thing) if 'scheme' in parsed: return len(parsed.scheme) >= 2 else: return len(parsed[0]) >= 2 + def load(resource): """ open a file or URL for reading. If the passed resource string is not a URL, or begins with 'file://', return a ``file``. Otherwise, return the result of urllib2.urlopen() """ import urllib2 @@ -438,9 +442,8 @@ def load(resource): if resource.startswith('file://'): resource = resource[len('file://'):] if not is_url(resource): # if no scheme is given, it is a file path return file(resource) return urllib2.urlopen(resource) -
--- a/testing/mozbase/mozfile/setup.py +++ b/testing/mozbase/mozfile/setup.py @@ -6,17 +6,17 @@ from setuptools import setup PACKAGE_NAME = 'mozfile' PACKAGE_VERSION = '1.2' setup(name=PACKAGE_NAME, version=PACKAGE_VERSION, description="Library of file utilities for use in Mozilla testing", long_description="see http://mozbase.readthedocs.org/", - classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla', author='Mozilla Automation and Tools team', author_email='tools@lists.mozilla.org', url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', license='MPL', packages=['mozfile'], include_package_data=True, zip_safe=False,
--- a/testing/mozbase/mozfile/tests/test_extract.py +++ b/testing/mozbase/mozfile/tests/test_extract.py @@ -113,17 +113,17 @@ class TestExtract(unittest.TestCase): mozfile.extract(filename, dest) except Exception as exception: pass finally: os.remove(filename) os.rmdir(dest) self.assertTrue(isinstance(exception, Exception)) - ### utility functions + # utility functions def create_tarball(self): """create a stub tarball for testing""" tempdir = stubs.create_stub() filename = tempfile.mktemp(suffix='.tar') archive = tarfile.TarFile(filename, mode='w') try: for path in stubs.files:
--- a/testing/mozbase/mozfile/tests/test_move_remove.py +++ b/testing/mozbase/mozfile/tests/test_move_remove.py @@ -21,16 +21,17 @@ def mark_readonly(path): :param path: path of directory/file of which modes must be changed """ mode = os.stat(path)[stat.ST_MODE] os.chmod(path, mode & ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH) class FileOpenCloseThread(threading.Thread): """Helper thread for asynchronous file handling""" + def __init__(self, path, delay, delete=False): threading.Thread.__init__(self) self.file_opened = threading.Event() self.delay = delay self.path = path self.delete = delete def run(self): @@ -196,16 +197,17 @@ class MozfileRemoveTestCase(unittest.Tes mozfile.remove(not_existing_path) except OSError as exc: if exc.errno == errno.ENOENT: self.fail("removing non existing path must not raise error") raise class MozFileMoveTestCase(unittest.TestCase): + def setUp(self): # Generate a stub self.tempdir = stubs.create_stub() self.addCleanup(mozfile.rmtree, self.tempdir) def test_move_file(self): file_path = os.path.join(self.tempdir, *stubs.files[1]) moved_path = file_path + '.moved'
--- a/testing/mozbase/mozhttpd/mozhttpd/__init__.py +++ b/testing/mozbase/mozhttpd/mozhttpd/__init__.py @@ -39,8 +39,10 @@ content from the current directory, defi 'function': resource_get } ]) print "Serving '%s' at %s:%s" % (httpd.docroot, httpd.host, httpd.port) httpd.start(block=True) """ from mozhttpd import MozHttpd, Request, RequestHandler, main from handlers import json_response + +__all__ = ['MozHttpd', 'Request', 'RequestHandler', 'main', 'json_response']
--- a/testing/mozbase/mozhttpd/mozhttpd/handlers.py +++ b/testing/mozbase/mozhttpd/mozhttpd/handlers.py @@ -1,15 +1,16 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import json + def json_response(func): """ Translates results of 'func' into a JSON response. """ def wrap(*a, **kw): (code, data) = func(*a, **kw) json_data = json.dumps(data) - return (code, { 'Content-type': 'application/json', - 'Content-Length': len(json_data) }, json_data) + return (code, {'Content-type': 'application/json', + 'Content-Length': len(json_data)}, json_data) return wrap
--- a/testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py +++ b/testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py @@ -15,16 +15,17 @@ import sys import os import urllib import urlparse import re import moznetwork import time from SocketServer import ThreadingMixIn + class EasyServer(ThreadingMixIn, BaseHTTPServer.HTTPServer): allow_reuse_address = True acceptable_errors = (errno.EPIPE, errno.ECONNABORTED) def handle_error(self, request, client_address): error = sys.exc_value if ((isinstance(error, socket.error) and @@ -57,31 +58,31 @@ class Request(object): if body_len and rfile: self.body = rfile.read(body_len) else: self.body = None class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): - docroot = os.getcwd() # current working directory at time of import + docroot = os.getcwd() # current working directory at time of import proxy_host_dirs = False request_log = [] log_requests = False request = None def __init__(self, *args, **kwargs): SimpleHTTPServer.SimpleHTTPRequestHandler.__init__(self, *args, **kwargs) self.extensions_map['.svg'] = 'image/svg+xml' def _try_handler(self, method): if self.log_requests: - self.request_log.append({ 'method': method, - 'path': self.request.path, - 'time': time.time() }) + self.request_log.append({'method': method, + 'path': self.request.path, + 'time': time.time()}) handlers = [handler for handler in self.urlhandlers if handler['method'] == method] for handler in handlers: m = re.match(handler['path'], self.request.path) if m: (response_code, headerdict, data) = \ handler['function'](self.request, *m.groups()) @@ -157,21 +158,21 @@ class RequestHandler(SimpleHTTPServer.Si # fragment and mangled the path for proxying, if required. path = posixpath.normpath(urllib.unquote(self.path)) words = path.split('/') words = filter(None, words) path = self.disk_root for word in words: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) - if word in (os.curdir, os.pardir): continue + if word in (os.curdir, os.pardir): + continue path = os.path.join(path, word) return path - # I found on my local network that calls to this were timing out # I believe all of these calls are from log_message def address_string(self): return "a.b.c.d" # This produces a LOT of noise def log_message(self, format, *args): pass @@ -256,27 +257,27 @@ class MozHttpd(object): server will be started on a separate thread that can be terminated by a call to stop(). """ self.httpd = EasyServer((self.host, self.port), self.handler_class) if block: self.httpd.serve_forever() else: self.server = threading.Thread(target=self.httpd.serve_forever) - self.server.setDaemon(True) # don't hang on exit + self.server.setDaemon(True) # don't hang on exit self.server.start() def stop(self): """ Stops the server. If the server is not running, this method has no effect. """ if self.httpd: - ### FIXME: There is no shutdown() method in Python 2.4... + # FIXME: There is no shutdown() method in Python 2.4... try: self.httpd.shutdown() except AttributeError: pass self.httpd = None def get_url(self, path="/"): """
--- a/testing/mozbase/mozhttpd/setup.py +++ b/testing/mozbase/mozhttpd/setup.py @@ -6,25 +6,24 @@ from setuptools import setup PACKAGE_VERSION = '0.7' deps = ['moznetwork >= 0.24'] setup(name='mozhttpd', version=PACKAGE_VERSION, description="Python webserver intended for use with Mozilla testing", long_description="see http://mozbase.readthedocs.org/", - classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla', author='Mozilla Automation and Testing Team', author_email='tools@lists.mozilla.org', url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', license='MPL', packages=['mozhttpd'], include_package_data=True, zip_safe=False, install_requires=deps, entry_points=""" # -*- Entry points: -*- [console_scripts] mozhttpd = mozhttpd:main """, ) -
--- a/testing/mozbase/mozhttpd/tests/api.py +++ b/testing/mozbase/mozhttpd/tests/api.py @@ -9,105 +9,106 @@ import mozhttpd import urllib2 import os import unittest import json import tempfile here = os.path.dirname(os.path.abspath(__file__)) + class ApiTest(unittest.TestCase): resource_get_called = 0 resource_post_called = 0 resource_del_called = 0 @mozhttpd.handlers.json_response def resource_get(self, request, objid): self.resource_get_called += 1 - return (200, { 'called': self.resource_get_called, - 'id': objid, - 'query': request.query }) + return (200, {'called': self.resource_get_called, + 'id': objid, + 'query': request.query}) @mozhttpd.handlers.json_response def resource_post(self, request): self.resource_post_called += 1 - return (201, { 'called': self.resource_post_called, - 'data': json.loads(request.body), - 'query': request.query }) + return (201, {'called': self.resource_post_called, + 'data': json.loads(request.body), + 'query': request.query}) @mozhttpd.handlers.json_response def resource_del(self, request, objid): self.resource_del_called += 1 - return (200, { 'called': self.resource_del_called, - 'id': objid, - 'query': request.query }) + return (200, {'called': self.resource_del_called, + 'id': objid, + 'query': request.query}) def get_url(self, path, server_port, querystr): url = "http://127.0.0.1:%s%s" % (server_port, path) if querystr: url += "?%s" % querystr return url def try_get(self, server_port, querystr): self.resource_get_called = 0 f = urllib2.urlopen(self.get_url('/api/resource/1', server_port, querystr)) try: self.assertEqual(f.getcode(), 200) except AttributeError: pass # python 2.4 - self.assertEqual(json.loads(f.read()), { 'called': 1, 'id': str(1), 'query': querystr }) + self.assertEqual(json.loads(f.read()), {'called': 1, 'id': str(1), 'query': querystr}) self.assertEqual(self.resource_get_called, 1) def try_post(self, server_port, querystr): self.resource_post_called = 0 - postdata = { 'hamburgers': '1234' } + postdata = {'hamburgers': '1234'} try: f = urllib2.urlopen(self.get_url('/api/resource/', server_port, querystr), data=json.dumps(postdata)) except urllib2.HTTPError as e: # python 2.4 self.assertEqual(e.code, 201) body = e.fp.read() else: self.assertEqual(f.getcode(), 201) body = f.read() - self.assertEqual(json.loads(body), { 'called': 1, - 'data': postdata, - 'query': querystr }) + self.assertEqual(json.loads(body), {'called': 1, + 'data': postdata, + 'query': querystr}) self.assertEqual(self.resource_post_called, 1) def try_del(self, server_port, querystr): self.resource_del_called = 0 opener = urllib2.build_opener(urllib2.HTTPHandler) request = urllib2.Request(self.get_url('/api/resource/1', server_port, querystr)) request.get_method = lambda: 'DEL' f = opener.open(request) try: self.assertEqual(f.getcode(), 200) except AttributeError: pass # python 2.4 - self.assertEqual(json.loads(f.read()), { 'called': 1, 'id': str(1), 'query': querystr }) + self.assertEqual(json.loads(f.read()), {'called': 1, 'id': str(1), 'query': querystr}) self.assertEqual(self.resource_del_called, 1) def test_api(self): httpd = mozhttpd.MozHttpd(port=0, - urlhandlers = [ { 'method': 'GET', - 'path': '/api/resource/([^/]+)/?', - 'function': self.resource_get }, - { 'method': 'POST', - 'path': '/api/resource/?', - 'function': self.resource_post }, - { 'method': 'DEL', - 'path': '/api/resource/([^/]+)/?', - 'function': self.resource_del } - ]) + urlhandlers=[{'method': 'GET', + 'path': '/api/resource/([^/]+)/?', + 'function': self.resource_get}, + {'method': 'POST', + 'path': '/api/resource/?', + 'function': self.resource_post}, + {'method': 'DEL', + 'path': '/api/resource/([^/]+)/?', + 'function': self.resource_del} + ]) httpd.start(block=False) server_port = httpd.httpd.server_port # GET self.try_get(server_port, '') self.try_get(server_port, '?foo=bar') @@ -164,48 +165,51 @@ class ApiTest(unittest.TestCase): opener.open(request) except urllib2.HTTPError: self.assertEqual(e.code, 404) exception_thrown = True self.assertTrue(exception_thrown) def test_api_with_docroot(self): httpd = mozhttpd.MozHttpd(port=0, docroot=here, - urlhandlers = [ { 'method': 'GET', - 'path': '/api/resource/([^/]+)/?', - 'function': self.resource_get } ]) + urlhandlers=[{'method': 'GET', + 'path': '/api/resource/([^/]+)/?', + 'function': self.resource_get}]) httpd.start(block=False) server_port = httpd.httpd.server_port # We defined a docroot, so we expect a directory listing f = urllib2.urlopen(self.get_url('/', server_port, None)) try: self.assertEqual(f.getcode(), 200) except AttributeError: pass # python 2.4 self.assertTrue('Directory listing for' in f.read()) # Make sure API methods still work self.try_get(server_port, '') self.try_get(server_port, '?foo=bar') + class ProxyTest(unittest.TestCase): def tearDown(self): # reset proxy opener in case it changed urllib2.install_opener(None) def test_proxy(self): docroot = tempfile.mkdtemp() self.addCleanup(mozfile.remove, docroot) hosts = ('mozilla.com', 'mozilla.org') unproxied_host = 'notmozilla.org' + def url(host): return 'http://%s/' % host index_filename = 'index.html' + def index_contents(host): return '%s index' % host index = file(os.path.join(docroot, index_filename), 'w') index.write(index_contents('*')) index.close() httpd = mozhttpd.MozHttpd(port=0, docroot=docroot) httpd.start(block=False)
--- a/testing/mozbase/mozhttpd/tests/baseurl.py +++ b/testing/mozbase/mozhttpd/tests/baseurl.py @@ -1,18 +1,19 @@ import mozhttpd import unittest + class BaseUrlTest(unittest.TestCase): def test_base_url(self): httpd = mozhttpd.MozHttpd(port=0) self.assertEqual(httpd.get_url(), None) httpd.start(block=False) self.assertEqual("http://127.0.0.1:%s/" % httpd.httpd.server_port, httpd.get_url()) - self.assertEqual("http://127.0.0.1:%s/cheezburgers.html" % \ - httpd.httpd.server_port, + self.assertEqual("http://127.0.0.1:%s/cheezburgers.html" % + httpd.httpd.server_port, httpd.get_url(path="/cheezburgers.html")) httpd.stop() if __name__ == '__main__': unittest.main()
--- a/testing/mozbase/mozhttpd/tests/filelisting.py +++ b/testing/mozbase/mozhttpd/tests/filelisting.py @@ -7,33 +7,35 @@ import mozhttpd import urllib2 import os import unittest import re here = os.path.dirname(os.path.abspath(__file__)) + class FileListingTest(unittest.TestCase): def check_filelisting(self, path=''): filelist = os.listdir(here) httpd = mozhttpd.MozHttpd(port=0, docroot=here) httpd.start(block=False) f = urllib2.urlopen("http://%s:%s/%s" % ('127.0.0.1', httpd.httpd.server_port, path)) for line in f.readlines(): - webline = re.sub('\<[a-zA-Z0-9\-\_\.\=\"\'\/\\\%\!\@\#\$\^\&\*\(\) ]*\>', '', line.strip('\n')).strip('/').strip().strip('@') + webline = re.sub('\<[a-zA-Z0-9\-\_\.\=\"\'\/\\\%\!\@\#\$\^\&\*\(\) ]*\>', + '', line.strip('\n')).strip('/').strip().strip('@') if webline and not webline.startswith("Directory listing for"): self.assertTrue(webline in filelist, "File %s in dir listing corresponds to a file" % webline) filelist.remove(webline) - self.assertFalse(filelist, "Should have no items in filelist (%s) unaccounted for" % filelist) - + self.assertFalse( + filelist, "Should have no items in filelist (%s) unaccounted for" % filelist) def test_filelist(self): self.check_filelisting() def test_filelist_params(self): self.check_filelisting('?foo=bar&fleem=&foo=fleem')
--- a/testing/mozbase/mozhttpd/tests/paths.py +++ b/testing/mozbase/mozhttpd/tests/paths.py @@ -4,17 +4,19 @@ # http://creativecommons.org/publicdomain/zero/1.0/ from mozfile import TemporaryDirectory import mozhttpd import os import unittest import urllib2 + class PathTest(unittest.TestCase): + def try_get(self, url, expected_contents): f = urllib2.urlopen(url) self.assertEqual(f.getcode(), 200) self.assertEqual(f.read(), expected_contents) def try_get_expect_404(self, url): with self.assertRaises(urllib2.HTTPError) as cm: urllib2.urlopen(url) @@ -37,17 +39,17 @@ class PathTest(unittest.TestCase): def test_substring_mappings(self): """Test that a path mapping that's a substring of another works.""" with TemporaryDirectory() as d1, TemporaryDirectory() as d2: open(os.path.join(d1, "test1.txt"), "w").write("test 1 contents") open(os.path.join(d2, "test2.txt"), "w").write("test 2 contents") httpd = mozhttpd.MozHttpd(port=0, path_mappings={'/abcxyz': d1, - '/abc': d2,} + '/abc': d2, } ) httpd.start(block=False) self.try_get(httpd.get_url("/abcxyz/test1.txt"), "test 1 contents") self.try_get(httpd.get_url("/abc/test2.txt"), "test 2 contents") httpd.stop() def test_multipart_path_mapping(self): """Test that a path mapping with multiple directories works."""
--- a/testing/mozbase/mozhttpd/tests/requestlog.py +++ b/testing/mozbase/mozhttpd/tests/requestlog.py @@ -4,16 +4,17 @@ import mozhttpd import urllib2 import os import unittest here = os.path.dirname(os.path.abspath(__file__)) + class RequestLogTest(unittest.TestCase): def check_logging(self, log_requests=False): httpd = mozhttpd.MozHttpd(port=0, docroot=here, log_requests=log_requests) httpd.start(block=False) url = "http://%s:%s/" % ('127.0.0.1', httpd.httpd.server_port) f = urllib2.urlopen(url)
--- a/testing/mozbase/mozinfo/mozinfo/__init__.py +++ b/testing/mozbase/mozinfo/mozinfo/__init__.py @@ -1,8 +1,9 @@ +# flake8: noqa # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import """ interface to transform introspected system information to a format palatable to
--- a/testing/mozbase/mozinfo/mozinfo/mozinfo.py +++ b/testing/mozbase/mozinfo/mozinfo/mozinfo.py @@ -15,33 +15,38 @@ import platform import re import sys from .string_version import StringVersion # keep a copy of the os module since updating globals overrides this _os = os + class unknown(object): """marker class for unknown information""" + def __nonzero__(self): return False + def __str__(self): return 'UNKNOWN' -unknown = unknown() # singleton +unknown = unknown() # singleton + def get_windows_version(): import ctypes + class OSVERSIONINFOEXW(ctypes.Structure): _fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong), ('dwMajorVersion', ctypes.c_ulong), ('dwMinorVersion', ctypes.c_ulong), ('dwBuildNumber', ctypes.c_ulong), ('dwPlatformId', ctypes.c_ulong), - ('szCSDVersion', ctypes.c_wchar*128), + ('szCSDVersion', ctypes.c_wchar * 128), ('wServicePackMajor', ctypes.c_ushort), ('wServicePackMinor', ctypes.c_ushort), ('wSuiteMask', ctypes.c_ushort), ('wProductType', ctypes.c_byte), ('wReserved', ctypes.c_byte)] os_version = OSVERSIONINFOEXW() os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) @@ -52,17 +57,17 @@ def get_windows_version(): return os_version.dwMajorVersion, os_version.dwMinorVersion, os_version.dwBuildNumber # get system information info = {'os': unknown, 'processor': unknown, 'version': unknown, 'os_version': unknown, 'bits': unknown, - 'has_sandbox': unknown } + 'has_sandbox': unknown} (system, node, release, version, machine, processor) = platform.uname() (bits, linkage) = platform.architecture() # get os information and related data if system in ["Microsoft", "Windows"]: info['os'] = 'win' # There is a Python bug on Windows to determine platform values # http://bugs.python.org/issue7860 @@ -133,17 +138,17 @@ if processor in ["i386", "i686"]: elif processor.upper() == "AMD64": bits = "64bit" processor = "x86_64" elif processor == "Power Macintosh": processor = "ppc" bits = re.search('(\d+)bit', bits).group(1) info.update({'processor': processor, 'bits': int(bits), - }) + }) if info['os'] == 'linux': import ctypes import errno PR_SET_SECCOMP = 22 SECCOMP_MODE_FILTER = 2 ctypes.CDLL("libc.so.6", use_errno=True).prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, 0) info['has_sandbox'] = ctypes.get_errno() == errno.EFAULT @@ -156,24 +161,26 @@ choices = {'os': ['linux', 'bsd', 'win', 'processor': ['x86', 'x86_64', 'ppc']} def sanitize(info): """Do some sanitization of input values, primarily to handle universal Mac builds.""" if "processor" in info and info["processor"] == "universal-x86-x86_64": # If we're running on OS X 10.6 or newer, assume 64-bit - if release[:4] >= "10.6": # Note this is a string comparison + if release[:4] >= "10.6": # Note this is a string comparison info["processor"] = "x86_64" info["bits"] = 64 else: info["processor"] = "x86" info["bits"] = 32 # method for updating information + + def update(new_info): """ Update the info. :param new_info: Either a dict containing the new info or a path/url to a json file containing the new info. """ @@ -188,19 +195,20 @@ def update(new_info): info.update(new_info) sanitize(info) globals().update(info) # convenience data for os access for os_name in choices['os']: globals()['is' + os_name.title()] = info['os'] == os_name # unix is special - if isLinux or isBsd: + if isLinux or isBsd: # noqa globals()['isUnix'] = True + def find_and_update_from_json(*dirs): """ Find a mozinfo.json file, load it, and update the info with the contents. :param dirs: Directories in which to look for the file. They will be searched after first looking in the root of the objdir if the current script is being run from a Mozilla objdir. @@ -224,36 +232,38 @@ def find_and_update_from_json(*dirs): d = _os.path.abspath(d) json_path = _os.path.join(d, "mozinfo.json") if _os.path.isfile(json_path): update(json_path) return json_path return None + def output_to_file(path): import json with open(path, 'w') as f: - f.write(json.dumps(info)); + f.write(json.dumps(info)) update({}) # exports __all__ = info.keys() __all__ += ['is' + os_name.title() for os_name in choices['os']] __all__ += [ 'info', 'unknown', 'main', 'choices', 'update', 'find_and_update_from_json', 'output_to_file', 'StringVersion', - ] +] + def main(args=None): # parse the command line from optparse import OptionParser parser = OptionParser(description=__doc__) for key in choices: parser.add_option('--%s' % key, dest=key, @@ -274,16 +284,17 @@ def main(args=None): # print out choices if requested flag = False for key, value in options.__dict__.items(): if value is True: print '%s choices: %s' % (key, ' '.join([str(choice) for choice in choices[key]])) flag = True - if flag: return + if flag: + return # otherwise, print out all info for key, value in info.items(): print '%s: %s' % (key, value) if __name__ == '__main__': main()
--- a/testing/mozbase/mozinfo/mozinfo/string_version.py +++ b/testing/mozbase/mozinfo/mozinfo/string_version.py @@ -4,16 +4,17 @@ from distutils.version import LooseVersion class StringVersion(str): """ A string version that can be compared with comparison operators. """ + def __init__(self, vstring): str.__init__(self, vstring) self.version = LooseVersion(vstring) def __repr__(self): return "StringVersion ('%s')" % self def __to_version(self, other):
--- a/testing/mozbase/mozinfo/setup.py +++ b/testing/mozbase/mozinfo/setup.py @@ -8,17 +8,17 @@ PACKAGE_VERSION = '0.9' # dependencies deps = ['mozfile >= 0.12'] setup(name='mozinfo', version=PACKAGE_VERSION, description="Library to get system information for use in Mozilla testing", long_description="see http://mozbase.readthedocs.org", - classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla', author='Mozilla Automation and Testing Team', author_email='tools@lists.mozilla.org', url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', license='MPL', packages=['mozinfo'], include_package_data=True, zip_safe=False,
--- a/testing/mozbase/mozinfo/tests/test.py +++ b/testing/mozbase/mozinfo/tests/test.py @@ -8,17 +8,19 @@ import json import mock import os import shutil import sys import tempfile import unittest import mozinfo + class TestMozinfo(unittest.TestCase): + def setUp(self): reload(mozinfo) self.tempdir = os.path.abspath(tempfile.mkdtemp()) # When running from an objdir mozinfo will use a build generated json file # instead of the ones created for testing. Prevent that from happening. # See bug 896038 for details. sys.modules['mozbuild'] = None @@ -43,20 +45,20 @@ class TestMozinfo(unittest.TestCase): j = os.path.join(self.tempdir, "mozinfo.json") with open(j, "w") as f: f.write(json.dumps({"foo": "xyz"})) mozinfo.update(j) self.assertEqual(mozinfo.info["foo"], "xyz") def test_update_file_invalid_json(self): """Test that mozinfo.update handles invalid JSON correctly""" - j = os.path.join(self.tempdir,'test.json') + j = os.path.join(self.tempdir, 'test.json') with open(j, 'w') as f: f.write('invalid{"json":') - self.assertRaises(ValueError,mozinfo.update,[j]) + self.assertRaises(ValueError, mozinfo.update, [j]) def test_find_and_update_file(self): """Test that mozinfo.find_and_update_from_json can find mozinfo.json in a directory passed to it.""" j = os.path.join(self.tempdir, "mozinfo.json") with open(j, "w") as f: f.write(json.dumps({"foo": "abcdefg"})) self.assertEqual(mozinfo.find_and_update_from_json(self.tempdir), j) @@ -65,17 +67,16 @@ class TestMozinfo(unittest.TestCase): def test_find_and_update_file_invalid_json(self): """Test that mozinfo.find_and_update_from_json can handle invalid JSON""" j = os.path.join(self.tempdir, "mozinfo.json") with open(j, 'w') as f: f.write('invalid{"json":') self.assertRaises(ValueError, mozinfo.find_and_update_from_json, self.tempdir) - def test_find_and_update_file_mozbuild(self): """Test that mozinfo.find_and_update_from_json can find mozinfo.json using the mozbuild module.""" j = os.path.join(self.tempdir, "mozinfo.json") with open(j, "w") as f: f.write(json.dumps({"foo": "123456"})) m = mock.MagicMock() # Mock the value of MozbuildObject.from_environment().topobjdir. @@ -87,16 +88,17 @@ class TestMozinfo(unittest.TestCase): def test_output_to_file(self): """Test that mozinfo.output_to_file works.""" path = os.path.join(self.tempdir, "mozinfo.json") mozinfo.output_to_file(path) self.assertEqual(open(path).read(), json.dumps(mozinfo.info)) class TestStringVersion(unittest.TestCase): + def test_os_version_is_a_StringVersion(self): self.assertIsInstance(mozinfo.os_version, mozinfo.StringVersion) def test_compare_to_string(self): version = mozinfo.StringVersion('10.10') self.assertGreater(version, '10.2') self.assertGreater('11', version)
--- a/testing/mozbase/mozinstall/mozinstall/__init__.py +++ b/testing/mozbase/mozinstall/mozinstall/__init__.py @@ -1,5 +1,6 @@ +# flake8: noqa # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. from mozinstall import *
--- a/testing/mozbase/mozinstall/mozinstall/mozinstall.py +++ b/testing/mozbase/mozinstall/mozinstall/mozinstall.py @@ -335,9 +335,8 @@ def uninstall_cli(argv=sys.argv[1:]): parser = OptionParser(usage="usage: %prog install_path") (options, args) = parser.parse_args(argv) if not len(args) == 1: parser.error('An installation path has to be specified.') # Run it uninstall(argv[0]) -
--- a/testing/mozbase/mozinstall/setup.py +++ b/testing/mozbase/mozinstall/setup.py @@ -10,41 +10,41 @@ try: description = file(os.path.join(here, 'README.md')).read() except IOError: description = None PACKAGE_VERSION = '1.12' deps = ['mozinfo >= 0.7', 'mozfile >= 1.0', - ] + ] setup(name='mozInstall', version=PACKAGE_VERSION, description="package for installing and uninstalling Mozilla applications", long_description="see http://mozbase.readthedocs.org/", # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=['Environment :: Console', 'Intended Audience :: Developers', 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Topic :: Software Development :: Libraries :: Python Modules', - ], + ], keywords='mozilla', author='Mozilla Automation and Tools team', author_email='tools@lists.mozilla.org', url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', license='MPL 2.0', packages=['mozinstall'], include_package_data=True, zip_safe=False, install_requires=deps, - tests_require=['mozprocess >= 0.15',], + tests_require=['mozprocess >= 0.15', ], # we have to generate two more executables for those systems that cannot run as Administrator # and the filename containing "install" triggers the UAC entry_points=""" # -*- Entry points: -*- [console_scripts] mozinstall = mozinstall:install_cli mozuninstall = mozinstall:uninstall_cli moz_add_to_system = mozinstall:install_cli
--- a/testing/mozbase/mozinstall/tests/test.py +++ b/testing/mozbase/mozinstall/tests/test.py @@ -9,16 +9,17 @@ import mozinstall import mozfile import os import tempfile import unittest # Store file location at load time here = os.path.dirname(os.path.abspath(__file__)) + class TestMozInstall(unittest.TestCase): @classmethod def setUpClass(cls): """ Setting up stub installers """ cls.dmg = os.path.join(here, 'Installer-Stubs', 'firefox.dmg') # XXX: We have removed firefox.exe since it is not valid for mozinstall 1.12 and higher # Bug 1157352 - We should grab a firefox.exe from the build process or download it @@ -27,69 +28,71 @@ class TestMozInstall(unittest.TestCase): cls.bz2 = os.path.join(here, 'Installer-Stubs', 'firefox.tar.bz2') def setUp(self): self.tempdir = tempfile.mkdtemp() def tearDown(self): mozfile.rmtree(self.tempdir) - @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe for mozinstall 1.12 and higher.") + @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe " + "for mozinstall 1.12 and higher.") def test_get_binary(self): """ Test mozinstall's get_binary method """ if mozinfo.isLinux: installdir = mozinstall.install(self.bz2, self.tempdir) binary = os.path.join(installdir, 'firefox') self.assertEqual(binary, mozinstall.get_binary(installdir, 'firefox')) elif mozinfo.isWin: installdir_exe = mozinstall.install(self.exe, os.path.join(self.tempdir, 'exe')) binary_exe = os.path.join(installdir_exe, 'core', 'firefox.exe') self.assertEqual(binary_exe, mozinstall.get_binary(installdir_exe, - 'firefox')) + 'firefox')) installdir_zip = mozinstall.install(self.zipfile, os.path.join(self.tempdir, 'zip')) binary_zip = os.path.join(installdir_zip, 'firefox.exe') self.assertEqual(binary_zip, mozinstall.get_binary(installdir_zip, - 'firefox')) + 'firefox')) elif mozinfo.isMac: installdir = mozinstall.install(self.dmg, self.tempdir) binary = os.path.join(installdir, 'Contents', 'MacOS', 'firefox') self.assertEqual(binary, mozinstall.get_binary(installdir, 'firefox')) def test_get_binary_error(self): """ Test an InvalidBinary error is raised """ tempdir_empty = tempfile.mkdtemp() self.assertRaises(mozinstall.InvalidBinary, mozinstall.get_binary, tempdir_empty, 'firefox') mozfile.rmtree(tempdir_empty) - @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe for mozinstall 1.12 and higher.") + @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe " + "for mozinstall 1.12 and higher.") def test_is_installer(self): """ Test we can identify a correct installer """ if mozinfo.isLinux: self.assertTrue(mozinstall.is_installer(self.bz2)) if mozinfo.isWin: # test zip installer self.assertTrue(mozinstall.is_installer(self.zipfile)) # test exe installer self.assertTrue(mozinstall.is_installer(self.exe)) try: # test stub browser file # without pefile on the system this test will fail - import pefile + import pefile # noqa stub_exe = os.path.join(here, 'build_stub', 'firefox.exe') self.assertFalse(mozinstall.is_installer(stub_exe)) except ImportError: pass if mozinfo.isMac: self.assertTrue(mozinstall.is_installer(self.dmg)) @@ -103,17 +106,18 @@ class TestMozInstall(unittest.TestCase): elif mozinfo.isWin: self.assertRaises(mozinstall.InvalidSource, mozinstall.install, self.bz2, 'firefox') elif mozinfo.isMac: self.assertRaises(mozinstall.InvalidSource, mozinstall.install, self.bz2, 'firefox') - @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe for mozinstall 1.12 and higher.") + @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe " + "for mozinstall 1.12 and higher.") def test_install(self): """ Test mozinstall's install capability """ if mozinfo.isLinux: installdir = mozinstall.install(self.bz2, self.tempdir) self.assertEqual(os.path.join(self.tempdir, 'firefox'), installdir) elif mozinfo.isWin: @@ -127,17 +131,18 @@ class TestMozInstall(unittest.TestCase): self.assertEqual(os.path.join(self.tempdir, 'zip', 'firefox'), installdir_zip) elif mozinfo.isMac: installdir = mozinstall.install(self.dmg, self.tempdir) self.assertEqual(os.path.join(os.path.realpath(self.tempdir), 'FirefoxStub.app'), installdir) - @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe for mozinstall 1.12 and higher.") + @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe " + "for mozinstall 1.12 and higher.") def test_uninstall(self): """ Test mozinstall's uninstall capabilites """ # Uninstall after installing if mozinfo.isLinux: installdir = mozinstall.install(self.bz2, self.tempdir) mozinstall.uninstall(installdir) self.assertFalse(os.path.exists(installdir))
--- a/testing/mozbase/mozleak/mozleak/__init__.py +++ b/testing/mozbase/mozleak/mozleak/__init__.py @@ -2,8 +2,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. """ mozleak is a library for extracting memory leaks from leak logs files. """ from .leaklog import process_leak_log + +__all__ = ['process_leak_log']
--- a/testing/mozbase/mozleak/mozleak/leaklog.py +++ b/testing/mozbase/mozleak/mozleak/leaklog.py @@ -1,19 +1,15 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. import os import re -import sys - -import mozinfo -import mozrunner.utils def _get_default_logger(): from mozlog import get_default_logger log = get_default_logger(component='mozleak') if not log: import logging @@ -66,18 +62,19 @@ def process_single_leak_file(leakLogFile if numLeaked != 0 or name == "TOTAL": log.info(line.rstrip()) # Analyse the leak log, but output later or it will interrupt the # leak table if name == "TOTAL": # Multiple default processes can end up writing their bloat views into a single # log, particularly on B2G. Eventually, these should be split into multiple # logs (bug 1068869), but for now, we report the largest leak. - if totalBytesLeaked != None: - leakAnalysis.append("WARNING | leakcheck | %s multiple BloatView byte totals found" + if totalBytesLeaked is not None: + leakAnalysis.append("WARNING | leakcheck | %s " + "multiple BloatView byte totals found" % processString) else: totalBytesLeaked = 0 if bytesLeaked > totalBytesLeaked: totalBytesLeaked = bytesLeaked # Throw out the information we had about the previous bloat # view. leakedObjectNames = [] @@ -189,17 +186,17 @@ def process_leak_log(leak_log_file, leak # are not going to produce leak logs we will ever see. knownProcessTypes = ["default", "plugin", "tab", "geckomediaplugin"] for processType in knownProcessTypes: log.info("TEST-INFO | leakcheck | %s process: leak threshold set at %d bytes" % (processType, leakThresholds.get(processType, 0))) for processType in leakThresholds: - if not processType in knownProcessTypes: + if processType not in knownProcessTypes: log.info("TEST-UNEXPECTED-FAIL | leakcheck | Unknown process type %s in leakThresholds" % processType) (leakLogFileDir, leakFileBase) = os.path.split(leakLogFile) if leakFileBase[-4:] == ".log": leakFileBase = leakFileBase[:-4] fileNameRegExp = re.compile(r"_([a-z]*)_pid\d*.log$") else: @@ -208,15 +205,15 @@ def process_leak_log(leak_log_file, leak for fileName in os.listdir(leakLogFileDir): if fileName.find(leakFileBase) != -1: thisFile = os.path.join(leakLogFileDir, fileName) m = fileNameRegExp.search(fileName) if m: processType = m.group(1) else: processType = "default" - if not processType in knownProcessTypes: + if processType not in knownProcessTypes: log.info("TEST-UNEXPECTED-FAIL | leakcheck | Leak log with unknown process type %s" % processType) leakThreshold = leakThresholds.get(processType, 0) process_single_leak_file(thisFile, processType, leakThreshold, processType in ignoreMissingLeaks, log=log, stackFixer=stack_fixer)
--- a/testing/mozbase/mozleak/setup.py +++ b/testing/mozbase/mozleak/setup.py @@ -9,17 +9,17 @@ PACKAGE_NAME = 'mozleak' PACKAGE_VERSION = '0.1' setup( name=PACKAGE_NAME, version=PACKAGE_VERSION, description="Library for extracting memory leaks from leak logs files", long_description="see http://mozbase.readthedocs.org/", - classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla', author='Mozilla Automation and Tools team', author_email='tools@lists.mozilla.org', url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', license='MPL', packages=['mozleak'], zip_safe=False, install_requires=[],
--- a/testing/mozbase/mozlog/mozlog/__init__.py +++ b/testing/mozbase/mozlog/mozlog/__init__.py @@ -19,8 +19,12 @@ from . import commandline from . import structuredlog from . import unstructured from .structuredlog import get_default_logger, set_default_logger from .proxy import get_proxy_logger # Backwards compatibility shim for consumers that use mozlog.structured structured = sys.modules[__name__] sys.modules['{}.structured'.format(__name__)] = structured + +__all__ = ['commandline', 'structuredlog', 'unstructured', + 'get_default_logger', 'set_default_logger', 'get_proxy_logger', + 'structured']
--- a/testing/mozbase/mozlog/mozlog/commandline.py +++ b/testing/mozbase/mozlog/mozlog/commandline.py @@ -20,37 +20,43 @@ log_formatters = { 'mach': (formatters.MachFormatter, "Human-readable output"), 'tbpl': (formatters.TbplFormatter, "TBPL style log format"), 'errorsummary': (formatters.ErrorSummaryFormatter, argparse.SUPPRESS), } TEXT_FORMATTERS = ('raw', 'mach') """a subset of formatters for non test harnesses related applications""" + def level_filter_wrapper(formatter, level): return handlers.LogLevelFilter(formatter, level) + def verbose_wrapper(formatter, verbose): formatter.verbose = verbose return formatter + def compact_wrapper(formatter, compact): formatter.compact = compact return formatter + def buffer_handler_wrapper(handler, buffer_limit): if buffer_limit == "UNLIMITED": buffer_limit = None else: buffer_limit = int(buffer_limit) return handlers.BufferHandler(handler, buffer_limit) + def valgrind_handler_wrapper(handler): return handlers.ValgrindHandler(handler) + def default_formatter_options(log_type, overrides): formatter_option_defaults = { "raw": { "level": "debug" } } rv = {"verbose": False, "level": "info"} @@ -66,17 +72,18 @@ fmt_options = { # "action" is used by the commandline parser in use. 'verbose': (verbose_wrapper, "Enables verbose mode for the given formatter.", ["mach"], "store_true"), 'compact': (compact_wrapper, "Enables compact mode for the given formatter.", ["tbpl"], "store_true"), 'level': (level_filter_wrapper, - "A least log level to subscribe to for the given formatter (debug, info, error, etc.)", + "A least log level to subscribe to for the given formatter " + "(debug, info, error, etc.)", ["mach", "raw", "tbpl"], "store"), 'buffer': (buffer_handler_wrapper, "If specified, enables message buffering at the given buffer size limit.", ["mach", "tbpl"], "store"), } def log_file(name): @@ -129,18 +136,18 @@ def add_logging_group(parser, include_fo opt_log_type = log_file group_add = group.add_argument for name, (cls, help_str) in log_formatters.iteritems(): if name in include_formatters: group_add("--log-" + name, action="append", type=opt_log_type, help=help_str) - for optname, (cls, help_str, formatters, action) in fmt_options.iteritems(): - for fmt in formatters: + for optname, (cls, help_str, formatters_, action) in fmt_options.iteritems(): + for fmt in formatters_: # make sure fmt is in log_formatters and is accepted if fmt in log_formatters and fmt in include_formatters: group_add("--log-%s-%s" % (fmt, optname), action=action, help=help_str, default=None) def setup_handlers(logger, formatters, formatter_options, allow_unused_options=False): """ @@ -177,17 +184,18 @@ def setup_handlers(logger, formatters, f for value in streams: handler = handlers.StreamHandler(stream=value, formatter=formatter) for wrapper, wrapper_args in handler_wrappers_and_options: handler = wrapper(handler, *wrapper_args) logger.add_handler(handler) -def setup_logging(logger, args, defaults=None, formatter_defaults=None, allow_unused_options=False): +def setup_logging(logger, args, defaults=None, formatter_defaults=None, + allow_unused_options=False): """ Configure a structuredlogger based on command line arguments. The created structuredlogger will also be set as the default logger, and can be retrieved with :py:func:`~mozlog.get_default_logger`. :param logger: A StructuredLogger instance or string name. If a string, a new StructuredLogger instance will be created using @@ -245,17 +253,17 @@ def setup_logging(logger, args, defaults formatters[formatter].append(value) if len(parts) == 3: _, formatter, opt = parts if formatter not in formatter_options: formatter_options[formatter] = default_formatter_options(formatter, formatter_defaults) formatter_options[formatter][opt] = values - #If there is no user-specified logging, go with the default options + # If there is no user-specified logging, go with the default options if not found: for name, value in defaults.iteritems(): formatters[name].append(value) elif not found_stdout_logger and sys.stdout in defaults.values(): for name, value in defaults.iteritems(): if value == sys.stdout: formatters[name].append(value)
--- a/testing/mozbase/mozlog/mozlog/formatters/__init__.py +++ b/testing/mozbase/mozlog/mozlog/formatters/__init__.py @@ -12,8 +12,12 @@ from errorsummary import ErrorSummaryFor try: import ujson as json except ImportError: import json def JSONFormatter(): return lambda x: json.dumps(x) + "\n" + +__all__ = ['UnittestFormatter', 'XUnitFormatter', 'HTMLFormatter', + 'MachFormatter', 'TbplFormatter', 'ErrorSummaryFormatter', + 'JSONFormatter']
--- a/testing/mozbase/mozlog/mozlog/formatters/base.py +++ b/testing/mozbase/mozlog/mozlog/formatters/base.py @@ -1,14 +1,15 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from ..reader import LogHandler + class BaseFormatter(LogHandler): """Base class for implementing non-trivial formatters. Subclasses are expected to provide a method for each action type they wish to handle, each taking a single argument for the test data. For example a trivial subclass that just produces the id of each test as it starts might be::
--- a/testing/mozbase/mozlog/mozlog/formatters/errorsummary.py +++ b/testing/mozbase/mozlog/mozlog/formatters/errorsummary.py @@ -1,17 +1,19 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import json from base import BaseFormatter + class ErrorSummaryFormatter(BaseFormatter): + def __init__(self): self.line_count = 0 def __call__(self, data): rv = BaseFormatter.__call__(self, data) self.line_count += 1 return rv
--- a/testing/mozbase/mozlog/mozlog/formatters/html/__init__.py +++ b/testing/mozbase/mozlog/mozlog/formatters/html/__init__.py @@ -1,1 +1,3 @@ from html import HTMLFormatter + +__all__ = ['HTMLFormatter']
--- a/testing/mozbase/mozlog/mozlog/formatters/html/html.py +++ b/testing/mozbase/mozlog/mozlog/formatters/html/html.py @@ -12,25 +12,27 @@ from .. import base from collections import defaultdict html = None raw = None base_path = os.path.split(__file__)[0] + def do_defered_imports(): global html global raw from .xmlgen import html, raw class HTMLFormatter(base.BaseFormatter): """Formatter that produces a simple HTML-formatted report.""" + def __init__(self): do_defered_imports() self.suite_name = None self.result_rows = [] self.test_count = defaultdict(int) self.start_times = {} self.suite_times = {"start": None, "end": None} @@ -47,39 +49,44 @@ class HTMLFormatter(base.BaseFormatter): html.style(raw(f.read()))) date_format = "%d %b %Y %H:%M:%S" version_info = data.get("version_info") if version_info: self.env["Device identifier"] = version_info.get("device_id") self.env["Device firmware (base)"] = version_info.get("device_firmware_version_base") self.env["Device firmware (date)"] = ( - datetime.utcfromtimestamp(int(version_info.get("device_firmware_date"))).strftime(date_format) if + datetime.utcfromtimestamp(int(version_info.get("device_firmware_date"))) + .strftime(date_format) if "device_firmware_date" in version_info else None) - self.env["Device firmware (incremental)"] = version_info.get("device_firmware_version_incremental") - self.env["Device firmware (release)"] = version_info.get("device_firmware_version_release") + self.env["Device firmware (incremental)"] = version_info.get( + "device_firmware_version_incremental") + self.env["Device firmware (release)"] = version_info.get( + "device_firmware_version_release") self.env["Gaia date"] = ( - datetime.utcfromtimestamp(int(version_info.get("gaia_date"))).strftime(date_format) if + datetime.utcfromtimestamp(int(version_info.get("gaia_date"))) + .strftime(date_format) if "gaia_date" in version_info else None) self.env["Gecko version"] = version_info.get("application_version") self.env["Gecko build"] = version_info.get("application_buildid") if version_info.get("application_changeset"): self.env["Gecko revision"] = version_info.get("application_changeset") if version_info.get("application_repository"): self.env["Gecko revision"] = html.a( version_info.get("application_changeset"), href="/".join([version_info.get("application_repository"), version_info.get("application_changeset")]), target="_blank") if version_info.get("gaia_changeset"): self.env["Gaia revision"] = html.a( version_info.get("gaia_changeset")[:12], - href="https://github.com/mozilla-b2g/gaia/commit/%s" % version_info.get("gaia_changeset"), + href="https://github.com/mozilla-b2g/gaia/commit/%s" % version_info.get( + "gaia_changeset"), target="_blank") device_info = data.get("device_info") if device_info: self.env["Device uptime"] = device_info.get("uptime") self.env["Device memory"] = device_info.get("memtotal") self.env["Device serial"] = device_info.get("id") @@ -97,18 +104,18 @@ class HTMLFormatter(base.BaseFormatter): tc_time = (data["time"] - self.start_times.pop(data["test"])) / 1000. additional_html = [] debug = data.get("extra", {}) # Add support for log exported from wptrunner. The structure of # reftest_screenshots is listed in wptrunner/executors/base.py. if debug.get('reftest_screenshots'): log_data = debug.get("reftest_screenshots", {}) debug = { - 'image1':'data:image/png;base64,' + log_data[0].get("screenshot", {}), - 'image2':'data:image/png;base64,' + log_data[2].get("screenshot", {}), + 'image1': 'data:image/png;base64,' + log_data[0].get("screenshot", {}), + 'image2': 'data:image/png;base64,' + log_data[2].get("screenshot", {}), 'differences': "Not Implemented", } links_html = [] status = status_name = data["status"] expected = data.get("expected", status) @@ -117,18 +124,18 @@ class HTMLFormatter(base.BaseFormatter): elif status not in ("PASS", "SKIP"): status_name = "EXPECTED_" + status self.test_count[status_name] += 1 if status in ['SKIP', 'FAIL', 'ERROR']: if debug.get('differences'): images = [ - ('image1','Image 1 (test)'), - ('image2','Image 2 (reference)') + ('image1', 'Image 1 (test)'), + ('image2', 'Image 2 (reference)') ] for title, description in images: screenshot = '%s' % debug[title] additional_html.append(html.div( html.a(html.img(src=screenshot), href="#"), html.br(), html.a(description), class_='screenshot')) @@ -191,35 +198,39 @@ class HTMLFormatter(base.BaseFormatter): self.head, html.body( html.script(raw(main_f.read())), html.p('Report generated on %s at %s' % ( generated.strftime('%d-%b-%Y'), generated.strftime('%H:%M:%S'))), html.h2('Environment'), html.table( - [html.tr(html.td(k), html.td(v)) for k, v in sorted(self.env.items()) if v], + [html.tr(html.td(k), html.td(v)) + for k, v in sorted(self.env.items()) if v], id='environment'), html.h2('Summary'), html.p('%i tests ran in %.1f seconds.' % (sum(self.test_count.itervalues()), (self.suite_times["end"] - self.suite_times["start"]) / 1000.), html.br(), html.span('%i passed' % self.test_count["PASS"], class_='pass'), ', ', html.span('%i skipped' % self.test_count["SKIP"], class_='skip'), ', ', - html.span('%i failed' % self.test_count["UNEXPECTED_FAIL"], class_='fail'), ', ', - html.span('%i errors' % self.test_count["UNEXPECTED_ERROR"], class_='error'), '.', + html.span('%i failed' % self.test_count[ + "UNEXPECTED_FAIL"], class_='fail'), ', ', + html.span('%i errors' % self.test_count[ + "UNEXPECTED_ERROR"], class_='error'), '.', html.br(), html.span('%i expected failures' % self.test_count["EXPECTED_FAIL"], class_='expected_fail'), ', ', html.span('%i unexpected passes' % self.test_count["UNEXPECTED_PASS"], class_='unexpected_pass'), '.'), html.h2('Results'), html.table([html.thead( html.tr([ html.th('Result', class_='sortable', col='result'), html.th('Test', class_='sortable', col='name'), html.th('Duration', class_='sortable numeric', col='duration'), html.th('Links')]), id='results-table-head'), - html.tbody(self.result_rows, id='results-table-body')], id='results-table'))) + html.tbody(self.result_rows, + id='results-table-body')], id='results-table'))) return u"<!DOCTYPE html>\n" + doc.unicode(indent=2)
--- a/testing/mozbase/mozlog/mozlog/formatters/html/xmlgen.py +++ b/testing/mozbase/mozlog/mozlog/formatters/html/xmlgen.py @@ -16,49 +16,55 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE F LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. This file is originally from: https://bitbucket.org/hpk42/py, specifically: https://bitbucket.org/hpk42/py/src/980c8d526463958ee7cae678a7e4e9b054f36b94/py/_xmlgen.py?at=default by holger krekel, holger at merlinux eu. 2009 """ -import sys, re +import sys +import re -if sys.version_info >= (3,0): +if sys.version_info >= (3, 0): def u(s): return s + def unicode(x): if hasattr(x, '__unicode__'): return x.__unicode__() return str(x) else: def u(s): return unicode(s) unicode = unicode class NamespaceMetaclass(type): + def __getattr__(self, name): if name[:1] == '_': raise AttributeError(name) if self == Namespace: raise ValueError("Namespace class is abstract") tagspec = self.__tagspec__ if tagspec is not None and name not in tagspec: raise AttributeError(name) classattr = {} if self.__stickyname__: classattr['xmlname'] = name cls = type(name, (self.__tagclass__,), classattr) setattr(self, name, cls) return cls + class Tag(list): + class Attr(object): + def __init__(self, **kwargs): self.__dict__.update(kwargs) def __init__(self, *args, **kwargs): super(Tag, self).__init__(args) self.attr = self.Attr(**kwargs) def __unicode__(self): @@ -75,58 +81,66 @@ class Tag(list): return "<%r tag object %d>" % (name, id(self)) Namespace = NamespaceMetaclass('Namespace', (object, ), { '__tagspec__': None, '__tagclass__': Tag, '__stickyname__': False, }) + class HtmlTag(Tag): + def unicode(self, indent=2): l = [] HtmlVisitor(l.append, indent, shortempty=False).visit(self) return u("").join(l) # exported plain html namespace + + class html(Namespace): __tagclass__ = HtmlTag __stickyname__ = True - __tagspec__ = dict([(x,1) for x in ( + __tagspec__ = dict([(x, 1) for x in ( 'a,abbr,acronym,address,applet,area,b,bdo,big,blink,' 'blockquote,body,br,button,caption,center,cite,code,col,' 'colgroup,comment,dd,del,dfn,dir,div,dl,dt,em,embed,' 'fieldset,font,form,frameset,h1,h2,h3,h4,h5,h6,head,html,' 'i,iframe,img,input,ins,kbd,label,legend,li,link,listing,' 'map,marquee,menu,meta,multicol,nobr,noembed,noframes,' 'noscript,object,ol,optgroup,option,p,pre,q,s,script,' 'select,small,span,strike,strong,style,sub,sup,table,' 'tbody,td,textarea,tfoot,th,thead,title,tr,tt,u,ul,xmp,' 'base,basefont,frame,hr,isindex,param,samp,var' ).split(',') if x]) class Style(object): + def __init__(self, **kw): for x, y in kw.items(): x = x.replace('_', '-') setattr(self, x, y) class raw(object): """just a box that can contain a unicode string that will be included directly in the output""" + def __init__(self, uniobj): self.uniobj = uniobj + class SimpleUnicodeVisitor(object): """ recursive visitor to write unicode. """ + def __init__(self, write, indent=0, curindent=0, shortempty=True): self.write = write self.cache = {} - self.visited = {} # for detection of recursion + self.visited = {} # for detection of recursion self.indent = indent self.curindent = curindent self.parents = [] self.shortempty = shortempty # short empty tags or not def visit(self, node): """ dispatcher on node's class/bases name. """ cls = node.__class__ @@ -140,17 +154,17 @@ class SimpleUnicodeVisitor(object): else: visitmethod = self.__object self.cache[cls] = visitmethod visitmethod(node) # the default fallback handler is marked private # to avoid clashes with the tag name object def __object(self, obj): - #self.write(obj) + # self.write(obj) self.write(escape(unicode(obj))) def raw(self, obj): self.write(obj.uniobj) def list(self, obj): assert id(obj) not in self.visited self.visited[id(obj)] = 1 @@ -172,17 +186,17 @@ class SimpleUnicodeVisitor(object): self.write(u('<%s%s>') % (tagname, self.attributes(tag))) self.parents.append(tag) for x in tag: self.visit(x) self.parents.pop() self.write(u('</%s>') % tagname) self.curindent -= self.indent else: - nameattr = tagname+self.attributes(tag) + nameattr = tagname + self.attributes(tag) if self._issingleton(tagname): self.write(u('<%s/>') % (nameattr,)) else: self.write(u('<%s></%s>') % (nameattr, tagname)) def attributes(self, tag): # serialize attributes attrlist = dir(tag.attr) @@ -208,57 +222,59 @@ class SimpleUnicodeVisitor(object): def getstyle(self, tag): """ return attribute list suitable for styling. """ try: styledict = tag.style.__dict__ except AttributeError: return [] else: - stylelist = [x+': ' + y for x,y in styledict.items()] + stylelist = [x + ': ' + y for x, y in styledict.items()] return [u(' style="%s"') % u('; ').join(stylelist)] def _issingleton(self, tagname): """can (and will) be overridden in subclasses""" return self.shortempty def _isinline(self, tagname): """can (and will) be overridden in subclasses""" return False + class HtmlVisitor(SimpleUnicodeVisitor): single = dict([(x, 1) for x in - ('br,img,area,param,col,hr,meta,link,base,' + ('br,img,area,param,col,hr,meta,link,base,' 'input,frame').split(',')]) inline = dict([(x, 1) for x in - ('a abbr acronym b basefont bdo big br cite code dfn em font ' - 'i img input kbd label q s samp select small span strike ' - 'strong sub sup textarea tt u var'.split(' '))]) + ('a abbr acronym b basefont bdo big br cite code dfn em font ' + 'i img input kbd label q s samp select small span strike ' + 'strong sub sup textarea tt u var'.split(' '))]) def repr_attribute(self, attrs, name): if name == 'class_': value = getattr(attrs, name) if value is None: return return super(HtmlVisitor, self).repr_attribute(attrs, name) def _issingleton(self, tagname): return tagname in self.single def _isinline(self, tagname): return tagname in self.inline class _escape: + def __init__(self): self.escape = { - u('"') : u('"'), u('<') : u('<'), u('>') : u('>'), - u('&') : u('&'), u("'") : u('''), - } + u('"'): u('"'), u('<'): u('<'), u('>'): u('>'), + u('&'): u('&'), u("'"): u('''), + } self.charef_rex = re.compile(u("|").join(self.escape.keys())) def _replacer(self, match): return self.escape[match.group(0)] def __call__(self, ustring): """ xml-escape the given unicode string. """ ustring = unicode(ustring)
--- a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py +++ b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py @@ -8,29 +8,34 @@ from collections import defaultdict try: import blessings except ImportError: blessings = None import base from .process import strstatus + def format_seconds(total): """Format number of seconds to MM:SS.DD form.""" minutes, seconds = divmod(total, 60) return '%2d:%05.2f' % (minutes, seconds) + class NullTerminal(object): + def __getattr__(self, name): return self._id def _id(self, value): return value + class MachFormatter(base.BaseFormatter): + def __init__(self, start_time=None, write_interval=False, write_times=True, terminal=None, disable_colors=False): if disable_colors: terminal = None elif terminal is None and blessings is not None: terminal = blessings.Terminal() @@ -128,17 +133,17 @@ class MachFormatter(base.BaseFormatter): else: rv.append("Ran %i tests" % self.summary_values["tests"]) rv.append("Expected results: %i" % self.summary_values["expected"]) unexpected_count = sum(self.summary_values["unexpected"].values()) if unexpected_count > 0: unexpected_str = " (%s)" % ", ".join("%s: %i" % (key, value) for key, value in - sorted(self.summary_values["unexpected"].items())) + sorted(self.summary_values["unexpected"].items())) else: unexpected_str = "" rv.append("Unexpected results: %i%s" % (unexpected_count, unexpected_str)) if self.summary_values["skipped"] > 0: rv.append("Skipped: %i" % self.summary_values["skipped"]) rv.append("") @@ -203,17 +208,17 @@ class MachFormatter(base.BaseFormatter): expected_str = "" test = self._get_test_id(data) if unexpected: self.summary_unexpected.append((test, unexpected)) self._update_summary(data) - #Reset the counts to 0 + # Reset the counts to 0 self.status_buffer[test] = {"count": 0, "unexpected": [], "pass": 0} self.has_unexpected[test] = bool(unexpected) if subtests["count"] != 0: rv = "Harness %s%s. Subtests passed %i/%i. Unexpected %s" % ( data["status"], expected_str, subtests["pass"], subtests["count"], len(unexpected)) else: @@ -353,39 +358,38 @@ class MachFormatter(base.BaseFormatter): if "stack" in data: rv += "\n%s" % data["stack"] return rv def lint(self, data): term = self.terminal if self.terminal is not None else NullTerminal() - fmt = "{path} {c1}{lineno}{column} {c2}{level}{normal} {message} {c1}{rule}({linter}){normal}" + fmt = "{path} {c1}{lineno}{column} {c2}{level}{normal} {message}" \ + " {c1}{rule}({linter}){normal}" message = fmt.format( path=data["path"], normal=term.normal, c1=term.grey, c2=term.red if data["level"] == 'error' else term.yellow, lineno=str(data["lineno"]), column=(":" + str(data["column"])) if data.get("column") else "", level=data["level"], message=data["message"], rule='{} '.format(data["rule"]) if data.get("rule") else "", linter=data["linter"].lower() if data.get("linter") else "", ) return message - def _get_subtest_data(self, data): test = self._get_test_id(data) return self.status_buffer.get(test, {"count": 0, "unexpected": [], "pass": 0}) def _time(self, data): entry_time = data["time"] if self.write_interval and self.last_time is not None: t = entry_time - self.last_time self.last_time = entry_time else: t = entry_time - self.start_time return t / 1000. -
--- a/testing/mozbase/mozlog/mozlog/formatters/process.py +++ b/testing/mozbase/mozlog/mozlog/formatters/process.py @@ -16,23 +16,23 @@ def strsig(n): global _SIG_NAME if _SIG_NAME is None: # cache signal names _SIG_NAME = {} for k in dir(signal): if (k.startswith("SIG") and not k.startswith("SIG_") - and k != "SIGCLD" and k != "SIGPOLL"): + and k != "SIGCLD" and k != "SIGPOLL"): _SIG_NAME[getattr(signal, k)] = k # Realtime signals mostly have no names if hasattr(signal, "SIGRTMIN") and hasattr(signal, "SIGRTMAX"): - for r in range(signal.SIGRTMIN+1, signal.SIGRTMAX+1): + for r in range(signal.SIGRTMIN + 1, signal.SIGRTMAX + 1): _SIG_NAME[r] = "SIGRTMIN+" + str(r - signal.SIGRTMIN) if n < 0 or n >= signal.NSIG: return "out-of-range signal, number %s" % n try: return _SIG_NAME[n] except KeyError: return "unrecognized signal, number %s" % n
--- a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py +++ b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py @@ -3,16 +3,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import functools from collections import deque from .base import BaseFormatter from .process import strstatus + def output_subtests(func): @functools.wraps(func) def inner(self, data): if self.subtests_count: return self._format_subtests(data.get("component")) + func(self, data) else: return func(self, data) return inner @@ -186,24 +187,26 @@ class TbplFormatter(BaseFormatter): message += "\n%s" % data["stack"] if message and message[-1] == "\n": message = message[:-1] extra = data.get("extra", {}) if "reftest_screenshots" in extra: screenshots = extra["reftest_screenshots"] if len(screenshots) == 3: - message += ("\nREFTEST IMAGE 1 (TEST): data:image/png;base64,%s\n" - "REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,%s") % (screenshots[0]["screenshot"], - screenshots[2]["screenshot"]) + message += ("\nREFTEST IMAGE 1 (TEST): data:image/png;base64,%s\n" + "REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,%s") % ( + screenshots[0]["screenshot"], + screenshots[2]["screenshot"]) elif len(screenshots) == 1: - message += "\nREFTEST IMAGE: data:image/png;base64,%(image1)s" % screenshots[0]["screenshot"] + message += "\nREFTEST IMAGE: data:image/png;base64,%(image1)s" \ + % screenshots[0]["screenshot"] failure_line = "TEST-UNEXPECTED-%s | %s | %s\n" % ( - data["status"], test_id, message) + data["status"], test_id, message) if data["expected"] not in ("PASS", "OK"): expected_msg = "expected %s | " % data["expected"] else: expected_msg = "" info_line = "TEST-INFO %s%s\n" % (expected_msg, duration_msg) return failure_line + info_line @@ -233,9 +236,9 @@ class TbplFormatter(BaseFormatter): rv = rv + line + "\n" return rv def lint(self, data): fmt = "TEST-UNEXPECTED-{level} | {path}:{lineno}{column} | {message} ({rule})" data["column"] = ":%s" % data["column"] if data["column"] else "" data['rule'] = data['rule'] or data['linter'] or "" - message.append(fmt.format(**data)) + return fmt.append(fmt.format(**data))
--- a/testing/mozbase/mozlog/mozlog/formatters/unittest.py +++ b/testing/mozbase/mozlog/mozlog/formatters/unittest.py @@ -1,18 +1,20 @@ #!/usr/bin/env python # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import base + class UnittestFormatter(base.BaseFormatter): """Formatter designed to produce output in a format like that used by the ``unittest`` module in the standard library.""" + def __init__(self): self.fails = [] self.errors = [] self.tests_run = 0 self.start_time = None self.end_time = None def suite_start(self, data):
--- a/testing/mozbase/mozlog/mozlog/formatters/xunit.py +++ b/testing/mozbase/mozlog/mozlog/formatters/xunit.py @@ -1,21 +1,22 @@ import types from xml.etree import ElementTree import base + def format_test_id(test_id): """Take a test id and return something that looks a bit like a class path""" if type(test_id) not in types.StringTypes: - #Not sure how to deal with reftests yet + # Not sure how to deal with reftests yet raise NotImplementedError - #Turn a path into something like a class heirachy + # Turn a path into something like a class heirachy return test_id.replace('.', '_').replace('/', ".") class XUnitFormatter(base.BaseFormatter): """Formatter that produces XUnit-style XML output. The tree is created in-memory so this formatter may be problematic with very large log files.
--- a/testing/mozbase/mozlog/mozlog/handlers/__init__.py +++ b/testing/mozbase/mozlog/mozlog/handlers/__init__.py @@ -1,8 +1,11 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from .base import LogLevelFilter, StreamHandler, BaseHandler from .statushandler import StatusHandler from .bufferhandler import BufferHandler from .valgrindhandler import ValgrindHandler + +__all__ = ['LogLevelFilter', 'StreamHandler', 'BaseHandler', + 'StatusHandler', 'BufferHandler', 'ValgrindHandler']
--- a/testing/mozbase/mozlog/mozlog/handlers/base.py +++ b/testing/mozbase/mozlog/mozlog/handlers/base.py @@ -46,24 +46,25 @@ class BaseHandler(object): class LogLevelFilter(BaseHandler): """Handler that filters out messages with action of log and a level lower than some specified level. :param inner: Handler to use for messages that pass this filter :param level: Minimum log level to process """ + def __init__(self, inner, level): BaseHandler.__init__(self, inner) self.inner = inner self.level = log_levels[level.upper()] def __call__(self, item): if (item["action"] != "log" or - log_levels[item["level"].upper()] <= self.level): + log_levels[item["level&q