author | Ted Mielczarek <ted.mielczarek@gmail.com> |
Tue, 21 Jun 2011 08:12:40 -0400 | |
changeset 72669 | 4edf5956780be0ab7bc2268adea9c87034c265e7 |
parent 72618 | ab19fc2d0dfe44ae5ca4a9412f3837c3506e0477 |
child 72670 | 6feef416f75294029efe5feec3832cc662a2313f |
push id | 45 |
push user | ffxbld |
push date | Thu, 22 Sep 2011 17:29:26 +0000 |
treeherder | mozilla-release@b3273da80b44 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | jmaher |
bugs | 664197 |
milestone | 7.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/build/Makefile.in +++ b/build/Makefile.in @@ -110,16 +110,21 @@ GARBAGE_DIRS += $(_VALGRIND_DIR) libs:: $(_VALGRIND_FILES) $(INSTALL) $^ $(_VALGRIND_DIR) ifdef ENABLE_TESTS libs:: $(topsrcdir)/tools/rb/fix_stack_using_bpsyms.py $(INSTALL) $< $(DIST)/bin +# Unit tests for ManifestParser +check:: + $(PYTHON) $(topsrcdir)/config/pythonpath.py -I$(srcdir) \ + $(srcdir)/tests/test.py + ifeq ($(OS_ARCH),Darwin) libs:: $(topsrcdir)/tools/rb/fix-macosx-stack.pl $(INSTALL) $< $(DIST)/bin libs:: $(topsrcdir)/tools/rb/fix_macosx_stack.py $(INSTALL) $< $(DIST)/bin # Basic unit tests for some stuff in the unify script check::
--- a/build/manifestparser.py +++ b/build/manifestparser.py @@ -39,30 +39,275 @@ """ Mozilla universal manifest parser """ # this file lives at # http://hg.mozilla.org/automation/ManifestDestiny/raw-file/tip/manifestparser.py -__all__ = ['ManifestParser', 'TestManifest', 'convert'] +__all__ = ['read_ini', # .ini reader + 'ManifestParser', 'TestManifest', 'convert', # manifest handling + 'parse', 'ParseError', 'ExpressionParser'] # conditional expression parser import os +import re import shutil import sys from fnmatch import fnmatch from optparse import OptionParser -version = '0.3.1' # package version +version = '0.5.1' # package version try: from setuptools import setup except ImportError: setup = None +# we need relpath, but it is introduced in python 2.6 +# http://docs.python.org/library/os.path.html +try: + relpath = os.path.relpath +except AttributeError: + def relpath(path, start): + """ + Return a relative version of a path + from /usr/lib/python2.6/posixpath.py + """ + + if not path: + raise ValueError("no path specified") + + start_list = os.path.abspath(start).split(os.path.sep) + path_list = os.path.abspath(path).split(os.path.sep) + + # Work out how much of the filepath is shared by start and path. + i = len(os.path.commonprefix([start_list, path_list])) + + rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:] + if not rel_list: + return start + return os.path.join(*rel_list) + +# expr.py +# from: +# http://k0s.org/mozilla/hg/expressionparser +# http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser + +# Implements a top-down parser/evaluator for simple boolean expressions. +# ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm +# +# Rough grammar: +# expr := literal +# | '(' expr ')' +# | expr '&&' expr +# | expr '||' expr +# | expr '==' expr +# | expr '!=' expr +# literal := BOOL +# | INT +# | STRING +# | IDENT +# BOOL := true|false +# INT := [0-9]+ +# STRING := "[^"]*" +# IDENT := [A-Za-z_]\w* + +# Identifiers take their values from a mapping dictionary passed as the second +# argument. + +# Glossary (see above URL for details): +# - nud: null denotation +# - led: left detonation +# - lbp: left binding power +# - rbp: right binding power + +class ident_token(object): + def __init__(self, value): + self.value = value + def nud(self, parser): + # identifiers take their value from the value mappings passed + # to the parser + return parser.value(self.value) + +class literal_token(object): + def __init__(self, value): + self.value = value + def nud(self, parser): + return self.value + +class eq_op_token(object): + "==" + def led(self, parser, left): + return left == parser.expression(self.lbp) + +class neq_op_token(object): + "!=" + def led(self, parser, left): + return left != parser.expression(self.lbp) + +class not_op_token(object): + "!" + def nud(self, parser): + return not parser.expression() + +class and_op_token(object): + "&&" + def led(self, parser, left): + right = parser.expression(self.lbp) + return left and right + +class or_op_token(object): + "||" + def led(self, parser, left): + right = parser.expression(self.lbp) + return left or right + +class lparen_token(object): + "(" + def nud(self, parser): + expr = parser.expression() + parser.advance(rparen_token) + return expr + +class rparen_token(object): + ")" + +class end_token(object): + """always ends parsing""" + +### derived literal tokens + +class bool_token(literal_token): + def __init__(self, value): + value = {'true':True, 'false':False}[value] + literal_token.__init__(self, value) + +class int_token(literal_token): + def __init__(self, value): + literal_token.__init__(self, int(value)) + +class string_token(literal_token): + def __init__(self, value): + literal_token.__init__(self, value[1:-1]) + +precedence = [(end_token, rparen_token), + (or_op_token,), + (and_op_token,), + (eq_op_token, neq_op_token), + (lparen_token,), + ] +for index, rank in enumerate(precedence): + for token in rank: + token.lbp = index # lbp = lowest left binding power + +class ParseError(Exception): + """errror parsing conditional expression""" + +class ExpressionParser(object): + def __init__(self, text, valuemapping, strict=False): + """ + Initialize the parser with input |text|, and |valuemapping| as + a dict mapping identifier names to values. + """ + self.text = text + self.valuemapping = valuemapping + self.strict = strict + + def _tokenize(self): + """ + Lex the input text into tokens and yield them in sequence. + """ + # scanner callbacks + def bool_(scanner, t): return bool_token(t) + def identifier(scanner, t): return ident_token(t) + def integer(scanner, t): return int_token(t) + def eq(scanner, t): return eq_op_token() + def neq(scanner, t): return neq_op_token() + def or_(scanner, t): return or_op_token() + def and_(scanner, t): return and_op_token() + def lparen(scanner, t): return lparen_token() + def rparen(scanner, t): return rparen_token() + def string_(scanner, t): return string_token(t) + def not_(scanner, t): return not_op_token() + + scanner = re.Scanner([ + (r"true|false", bool_), + (r"[a-zA-Z_]\w*", identifier), + (r"[0-9]+", integer), + (r'("[^"]*")|(\'[^\']*\')', string_), + (r"==", eq), + (r"!=", neq), + (r"\|\|", or_), + (r"!", not_), + (r"&&", and_), + (r"\(", lparen), + (r"\)", rparen), + (r"\s+", None), # skip whitespace + ]) + tokens, remainder = scanner.scan(self.text) + for t in tokens: + yield t + yield end_token() + + def value(self, ident): + """ + Look up the value of |ident| in the value mapping passed in the + constructor. + """ + if self.strict: + return self.valuemapping[ident] + else: + return self.valuemapping.get(ident, None) + + def advance(self, expected): + """ + Assert that the next token is an instance of |expected|, and advance + to the next token. + """ + if not isinstance(self.token, expected): + raise Exception, "Unexpected token!" + self.token = self.iter.next() + + def expression(self, rbp=0): + """ + Parse and return the value of an expression until a token with + right binding power greater than rbp is encountered. + """ + t = self.token + self.token = self.iter.next() + left = t.nud(self) + while rbp < self.token.lbp: + t = self.token + self.token = self.iter.next() + left = t.led(self, left) + return left + + def parse(self): + """ + Parse and return the value of the expression in the text + passed to the constructor. Raises a ParseError if the expression + could not be parsed. + """ + try: + self.iter = self._tokenize() + self.token = self.iter.next() + return self.expression() + except: + raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping)) + + __call__ = parse + +def parse(text, **values): + """ + Parse and evaluate a boolean expression in |text|. Use |values| to look + up the value of identifiers referenced in the expression. Returns the final + value of the expression. A ParseError will be raised if parsing fails. + """ + return ExpressionParser(text, values).parse() + def normalize_path(path): """normalize a relative path""" if sys.platform.startswith('win'): return path.replace('/', os.path.sep) return path def read_ini(fp, variables=None, default='DEFAULT', comments=';#', separators=('=', ':'), @@ -121,17 +366,17 @@ def read_ini(fp, variables=None, default section_names.add(section) current_section = {} sections.append((section, current_section)) continue # if there aren't any sections yet, something bad happen if not section_names: - raise Exception('No sections yet :(') + raise Exception('No sections found') # (key, value) pair for separator in separators: if separator in stripped: key, value = stripped.split(separator, 1) key = key.strip() value = value.strip() @@ -169,16 +414,17 @@ class ManifestParser(object): ### methods for reading manifests def __init__(self, manifests=(), defaults=None, strict=True): self._defaults = defaults or {} self.tests = [] self.strict = strict self.rootdir = None + self.relativeRoot = None if manifests: self.read(*manifests) def getRelativeRoot(self, root): return root def read(self, *filenames, **defaults): @@ -237,61 +483,64 @@ class ManifestParser(object): path = os.path.join(here, path) test['path'] = path # append the item self.tests.append(test) ### methods for querying manifests - def query(self, *checks): + def query(self, *checks, **kw): """ general query function for tests - checks : callable conditions to test if the test fulfills the query """ + tests = kw.get('tests', None) + if tests is None: + tests = self.tests retval = [] - for test in self.tests: + for test in tests: for check in checks: if not check(test): break else: retval.append(test) return retval - def get(self, _key=None, inverse=False, tags=None, **kwargs): + def get(self, _key=None, inverse=False, tags=None, tests=None, **kwargs): # TODO: pass a dict instead of kwargs since you might hav # e.g. 'inverse' as a key in the dict # TODO: tags should just be part of kwargs with None values # (None == any is kinda weird, but probably still better) # fix up tags if tags: tags = set(tags) else: tags = set() # make some check functions if inverse: - has_tags = lambda test: tags.isdisjoint(test.keys()) + has_tags = lambda test: not tags.intersection(test.keys()) def dict_query(test): for key, value in kwargs.items(): if test.get(key) == value: return False return True else: has_tags = lambda test: tags.issubset(test.keys()) def dict_query(test): for key, value in kwargs.items(): if test.get(key) != value: return False return True # query the tests - tests = self.query(has_tags, dict_query) + tests = self.query(has_tags, dict_query, tests=tests) # if a key is given, return only a list of that key # useful for keys like 'name' or 'path' if _key: return [test[_key] for test in tests] # return the tests return tests @@ -360,17 +609,17 @@ class ManifestParser(object): print >> fp, '%s = %s' % (key, value) print >> fp for test in tests: test = test.copy() # don't overwrite path = test['name'] if not os.path.isabs(path): - path = os.path.relpath(test['path'], self.rootdir) + path = relpath(test['path'], self.rootdir) print >> fp, '[%s]' % path # reserved keywords: reserved = ['path', 'name', 'here', 'manifest'] for key in sorted(test.keys()): if key in reserved: continue if key in global_kwargs: @@ -405,17 +654,17 @@ class ManifestParser(object): if not tests: return # nothing to do! # root directory if rootdir is None: rootdir = self.rootdir # copy the manifests + tests - manifests = [os.path.relpath(manifest, rootdir) for manifest in self.manifests()] + manifests = [relpath(manifest, rootdir) for manifest in self.manifests()] for manifest in manifests: destination = os.path.join(directory, manifest) dirname = os.path.dirname(destination) if not os.path.exists(dirname): os.makedirs(dirname) else: # sanity check assert os.path.isdir(dirname) @@ -423,17 +672,17 @@ class ManifestParser(object): for test in tests: if os.path.isabs(test['name']): continue source = test['path'] if not os.path.exists(source): print >> sys.stderr, "Missing test: '%s' does not exist!" % source continue # TODO: should err on strict - destination = os.path.join(directory, os.path.relpath(test['path'], rootdir)) + destination = os.path.join(directory, relpath(test['path'], rootdir)) shutil.copy(source, destination) # TODO: ensure that all of the tests are below the from_dir def update(self, from_dir, rootdir=None, *tags, **kwargs): """ update the tests as listed in a manifest from a directory - from_dir : directory where the tests live - rootdir : root directory to copy to (if not given from manifests) @@ -446,82 +695,89 @@ class ManifestParser(object): # get the root directory if not rootdir: rootdir = self.rootdir # copy them! for test in tests: if not os.path.isabs(test['name']): - relpath = os.path.relpath(test['path'], rootdir) - source = os.path.join(from_dir, relpath) + _relpath = relpath(test['path'], rootdir) + source = os.path.join(from_dir, _relpath) if not os.path.exists(source): # TODO err on strict print >> sys.stderr, "Missing test: '%s'; skipping" % test['name'] continue - destination = os.path.join(rootdir, relpath) + destination = os.path.join(rootdir, _relpath) shutil.copy(source, destination) class TestManifest(ManifestParser): """ apply logic to manifests; this is your integration layer :) specific harnesses may subclass from this if they need more logic """ - def filter(self, tag, value, tests=None): + def filter(self, values, tests): """ filter on a specific list tag, e.g.: run-if.os = win linux skip-if.os = mac """ - if tests is None: - tests = self.tests - # tags: - run_tag = 'run-if.' + tag - skip_tag = 'skip-if.' + tag + run_tag = 'run-if' + skip_tag = 'skip-if' + fail_tag = 'fail-if' # loop over test for test in tests: reason = None # reason to disable # tagged-values to run if run_tag in test: - values = test[run_tag].split() - if value not in values: - reason = '%s %s not in run values %s' % (tag, value, values) + condition = test[run_tag] + if not parse(condition, **values): + reason = '%s: %s' % (run_tag, condition) # tagged-values to skip if skip_tag in test: - values = test[skip_tag].split() - if value in values: - reason = '%s %s in skipped values %s' % (tag, value, values) + condition = test[skip_tag] + if parse(condition, **values): + reason = '%s: %s' % (skip_tag, condition) # mark test as disabled if there's a reason if reason: test.setdefault('disabled', reason) - def active_tests(self, exists=True, disabled=True, **tags): + # mark test as a fail if so indicated + if fail_tag in test: + condition = test[fail_tag] + if parse(condition, **values): + test['expected'] = 'fail' + + def active_tests(self, exists=True, disabled=True, **values): """ - exists : return only existing tests - disabled : whether to return disabled tests - tags : keys and values to filter on (e.g. `os = linux mac`) """ tests = [i.copy() for i in self.tests] # shallow copy + + # mark all tests as passing unless indicated otherwise + for test in tests: + test['expected'] = test.get('expected', 'pass') # ignore tests that do not exist if exists: tests = [test for test in tests if os.path.exists(test['path'])] # filter by tags - for tag, value in tags.items(): - self.filter(tag, value, tests) + self.filter(values, tests) # ignore disabled tests if specified if not disabled: tests = [test for test in tests if not 'disabled' in test] # return active tests return tests @@ -758,17 +1014,17 @@ class SetupCLI(CLICommand): filename = os.path.join(here, 'README.txt') description = file(filename).read() except: description = '' os.chdir(here) setup(name='ManifestDestiny', version=version, - description="universal reader for manifests", + description="Universal manifests for Mozilla test harnesses", long_description=description, classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers keywords='mozilla manifests', author='Jeff Hammel', author_email='jhammel@mozilla.com', url='https://wiki.mozilla.org/Auto-tools/Projects/ManifestDestiny', license='MPL', zip_safe=False,
new file mode 100755 --- /dev/null +++ b/build/mozinfo.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python + +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is Mozilla Corporation Code. +# +# The Initial Developer of the Original Code is +# Mikeal Rogers. +# Portions created by the Initial Developer are Copyright (C) 2008 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Mikeal Rogers <mikeal.rogers@gmail.com> +# Henrik Skupin <hskupin@mozilla.com> +# Clint Talbert <ctalbert@mozilla.com> +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +""" +file for interface to transform introspected system information to a format +pallatable to Mozilla + +Information: +- os : what operating system ['win', 'mac', 'linux', ...] +- bits : 32 or 64 +- processor : processor architecture ['x86', 'x86_64', 'ppc', ...] +- version : operating system version string + +For windows, the service pack information is also included +""" + +# TODO: it might be a good idea of adding a system name (e.g. 'Ubuntu' for +# linux) to the information; I certainly wouldn't want anyone parsing this +# information and having behaviour depend on it + +import os +import platform +import re +import sys + +# keep a copy of the os module since updating globals overrides this +_os = os + +class unknown(object): + """marker class for unknown information""" + def __nonzero__(self): + return False + def __str__(self): + return 'UNKNOWN' +unknown = unknown() # singleton + +# get system information + +info = {'os': unknown, + 'processor': unknown, + 'version': unknown, + 'bits': unknown } + +(system, node, release, version, machine, processor) = platform.uname() +(bits, linkage) = platform.architecture() + +if system in ["Microsoft", "Windows"]: + info['os'] = 'win' + + # There is a Python bug on Windows to determine platform values + # http://bugs.python.org/issue7860 + if "PROCESSOR_ARCHITEW6432" in os.environ: + processor = os.environ.get("PROCESSOR_ARCHITEW6432", processor) + else: + processor = os.environ.get('PROCESSOR_ARCHITECTURE', processor) + system = os.environ.get("OS", system).replace('_', ' ') + service_pack = os.sys.getwindowsversion()[4] + info['service_pack'] = service_pack +elif system == "Linux": + (distro, version, codename) = platform.dist() + version = distro + " " + version + if not processor: + processor = machine + info['os'] = 'linux' +elif system == "Darwin": + (release, versioninfo, machine) = platform.mac_ver() + version = "OS X " + release + info['os'] = 'mac' +elif sys.platform in ('solaris', 'sunos5'): + info['os'] = 'unix' + version = sys.platform + +# processor type and bits +if processor in ["i386", "i686"]: + if bits == "32bit": + processor = "x86" + elif bits == "64bit": + processor = "x86_64" +elif processor == "AMD64": + bits = "64bit" + processor = "x86_64" +elif processor == "Power Macintosh": + processor = "ppc" +bits = re.search('(\d+)bit', bits).group(1) + +info.update({'version': version, + 'processor': processor, + 'bits': int(bits), + }) + +def update(new_info): + """update the info""" + info.update(new_info) + globals().update(info) + +update({}) + +choices = {'os': ['linux', 'win', 'mac', 'unix'], + 'bits': [32, 64], + 'processor': ['x86', 'x86_64', 'ppc']} + +# exports +__all__ = info.keys() +__all__ += ['info', 'unknown', 'main', 'choices'] + + +def main(args=None): + + # parse the command line + from optparse import OptionParser + parser = OptionParser() + for key in choices: + parser.add_option('--%s' % key, dest=key, + action='store_true', default=False, + help="display choices for %s" % key) + options, args = parser.parse_args() + + # args are JSON blobs to override info + if args: + try: + from json import loads + except ImportError: + try: + from simplejson import loads + except ImportError: + def loads(string): + """*really* simple json; will not work with unicode""" + return eval(string, {'true': True, 'false': False, 'null': None}) + for arg in args: + if _os.path.exists(arg): + string = file(arg).read() + else: + string = arg + update(loads(string)) + + # print out choices if requested + flag = False + for key, value in options.__dict__.items(): + if value is True: + print '%s choices: %s' % (key, ' '.join([str(choice) + for choice in choices[key]])) + flag = True + if flag: return + + # otherwise, print out all info + for key, value in info.items(): + print '%s: %s' % (key, value) + +if __name__ == '__main__': + main()
new file mode 100644 --- /dev/null +++ b/build/tests/filter-example.ini @@ -0,0 +1,11 @@ +# illustrate test filters based on various categories + +[windowstest] +run-if = os == 'win' + +[fleem] +skip-if = os == 'mac' + +[linuxtest] +skip-if = (os == 'mac') || (os == 'win') +fail-if = toolkit == 'cocoa'
new file mode 100644 --- /dev/null +++ b/build/tests/fleem @@ -0,0 +1,1 @@ +# dummy spot for "fleem" test
new file mode 100644 --- /dev/null +++ b/build/tests/include-example.ini @@ -0,0 +1,11 @@ +[DEFAULT] +foo = bar + +[include:include/bar.ini] + +[fleem] + +[include:include/foo.ini] +red = roses +blue = violets +yellow = daffodils \ No newline at end of file
new file mode 100644 --- /dev/null +++ b/build/tests/include/bar.ini @@ -0,0 +1,4 @@ +[DEFAULT] +foo = fleem + +[crash-handling] \ No newline at end of file
new file mode 100644 --- /dev/null +++ b/build/tests/include/crash-handling @@ -0,0 +1,1 @@ +# dummy spot for "crash-handling" test
new file mode 100644 --- /dev/null +++ b/build/tests/include/flowers @@ -0,0 +1,1 @@ +# dummy spot for "flowers" test
new file mode 100644 --- /dev/null +++ b/build/tests/include/foo.ini @@ -0,0 +1,5 @@ +[DEFAULT] +blue = ocean + +[flowers] +yellow = submarine \ No newline at end of file
new file mode 100644 --- /dev/null +++ b/build/tests/mozmill-example.ini @@ -0,0 +1,80 @@ +[testAddons/testDisableEnablePlugin.js] +[testAddons/testGetAddons.js] +[testAddons/testSearchAddons.js] +[testAwesomeBar/testAccessLocationBar.js] +[testAwesomeBar/testCheckItemHighlight.js] +[testAwesomeBar/testEscapeAutocomplete.js] +[testAwesomeBar/testFaviconInAutocomplete.js] +[testAwesomeBar/testGoButton.js] +[testAwesomeBar/testLocationBarSearches.js] +[testAwesomeBar/testPasteLocationBar.js] +[testAwesomeBar/testSuggestHistoryBookmarks.js] +[testAwesomeBar/testVisibleItemsMax.js] +[testBookmarks/testAddBookmarkToMenu.js] +[testCookies/testDisableCookies.js] +[testCookies/testEnableCookies.js] +[testCookies/testRemoveAllCookies.js] +[testCookies/testRemoveCookie.js] +[testDownloading/testCloseDownloadManager.js] +[testDownloading/testDownloadStates.js] +[testDownloading/testOpenDownloadManager.js] +[testFindInPage/testFindInPage.js] +[testFormManager/testAutoCompleteOff.js] +[testFormManager/testBasicFormCompletion.js] +[testFormManager/testClearFormHistory.js] +[testFormManager/testDisableFormManager.js] +[testGeneral/testGoogleSuggestions.js] +[testGeneral/testStopReloadButtons.js] +[testInstallation/testBreakpadInstalled.js] +[testLayout/testNavigateFTP.js] +[testPasswordManager/testPasswordNotSaved.js] +[testPasswordManager/testPasswordSavedAndDeleted.js] +[testPopups/testPopupsAllowed.js] +[testPopups/testPopupsBlocked.js] +[testPreferences/testPaneRetention.js] +[testPreferences/testPreferredLanguage.js] +[testPreferences/testRestoreHomepageToDefault.js] +[testPreferences/testSetToCurrentPage.js] +[testPreferences/testSwitchPanes.js] +[testPrivateBrowsing/testAboutPrivateBrowsing.js] +[testPrivateBrowsing/testCloseWindow.js] +[testPrivateBrowsing/testDisabledElements.js] +[testPrivateBrowsing/testDisabledPermissions.js] +[testPrivateBrowsing/testDownloadManagerClosed.js] +[testPrivateBrowsing/testGeolocation.js] +[testPrivateBrowsing/testStartStopPBMode.js] +[testPrivateBrowsing/testTabRestoration.js] +[testPrivateBrowsing/testTabsDismissedOnStop.js] +[testSearch/testAddMozSearchProvider.js] +[testSearch/testFocusAndSearch.js] +[testSearch/testGetMoreSearchEngines.js] +[testSearch/testOpenSearchAutodiscovery.js] +[testSearch/testRemoveSearchEngine.js] +[testSearch/testReorderSearchEngines.js] +[testSearch/testRestoreDefaults.js] +[testSearch/testSearchSelection.js] +[testSearch/testSearchSuggestions.js] +[testSecurity/testBlueLarry.js] +[testSecurity/testDefaultPhishingEnabled.js] +[testSecurity/testDefaultSecurityPrefs.js] +[testSecurity/testEncryptedPageWarning.js] +[testSecurity/testGreenLarry.js] +[testSecurity/testGreyLarry.js] +[testSecurity/testIdentityPopupOpenClose.js] +[testSecurity/testSSLDisabledErrorPage.js] +[testSecurity/testSafeBrowsingNotificationBar.js] +[testSecurity/testSafeBrowsingWarningPages.js] +[testSecurity/testSecurityInfoViaMoreInformation.js] +[testSecurity/testSecurityNotification.js] +[testSecurity/testSubmitUnencryptedInfoWarning.js] +[testSecurity/testUnknownIssuer.js] +[testSecurity/testUntrustedConnectionErrorPage.js] +[testSessionStore/testUndoTabFromContextMenu.js] +[testTabbedBrowsing/testBackgroundTabScrolling.js] +[testTabbedBrowsing/testCloseTab.js] +[testTabbedBrowsing/testNewTab.js] +[testTabbedBrowsing/testNewWindow.js] +[testTabbedBrowsing/testOpenInBackground.js] +[testTabbedBrowsing/testOpenInForeground.js] +[testTechnicalTools/testAccessPageInfoDialog.js] +[testToolbar/testBackForwardButtons.js]
new file mode 100644 --- /dev/null +++ b/build/tests/mozmill-restart-example.ini @@ -0,0 +1,26 @@ +[DEFAULT] +type = restart + +[restartTests/testExtensionInstallUninstall/test2.js] +foo = bar + +[restartTests/testExtensionInstallUninstall/test1.js] +foo = baz + +[restartTests/testExtensionInstallUninstall/test3.js] +[restartTests/testSoftwareUpdateAutoProxy/test2.js] +[restartTests/testSoftwareUpdateAutoProxy/test1.js] +[restartTests/testMasterPassword/test1.js] +[restartTests/testExtensionInstallGetAddons/test2.js] +[restartTests/testExtensionInstallGetAddons/test1.js] +[restartTests/testMultipleExtensionInstallation/test2.js] +[restartTests/testMultipleExtensionInstallation/test1.js] +[restartTests/testThemeInstallUninstall/test2.js] +[restartTests/testThemeInstallUninstall/test1.js] +[restartTests/testThemeInstallUninstall/test3.js] +[restartTests/testDefaultBookmarks/test1.js] +[softwareUpdate/testFallbackUpdate/test2.js] +[softwareUpdate/testFallbackUpdate/test1.js] +[softwareUpdate/testFallbackUpdate/test3.js] +[softwareUpdate/testDirectUpdate/test2.js] +[softwareUpdate/testDirectUpdate/test1.js]
new file mode 100644 --- /dev/null +++ b/build/tests/path-example.ini @@ -0,0 +1,2 @@ +[foo] +path = fleem \ No newline at end of file
new file mode 100755 --- /dev/null +++ b/build/tests/test.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is mozilla.org code. +# +# The Initial Developer of the Original Code is +# Mozilla.org. +# Portions created by the Initial Developer are Copyright (C) 2010 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Jeff Hammel <jhammel@mozilla.com> (Original author) +# +# Alternatively, the contents of this file may be used under the terms of +# either of the GNU General Public License Version 2 or later (the "GPL"), +# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +"""tests for ManifestDestiny""" + +import doctest +import os +import sys +from optparse import OptionParser + +def run_tests(raise_on_error=False, report_first=False): + + # add results here + results = {} + + # doctest arguments + directory = os.path.dirname(os.path.abspath(__file__)) + extraglobs = {} + doctest_args = dict(extraglobs=extraglobs, + module_relative=False, + raise_on_error=raise_on_error) + if report_first: + doctest_args['optionflags'] = doctest.REPORT_ONLY_FIRST_FAILURE + + # gather tests + directory = os.path.dirname(os.path.abspath(__file__)) + tests = [ test for test in os.listdir(directory) + if test.endswith('.txt') and test.startswith('test_')] + os.chdir(directory) + + # run the tests + for test in tests: + try: + results[test] = doctest.testfile(test, **doctest_args) + except doctest.DocTestFailure, failure: + raise + except doctest.UnexpectedException, failure: + raise failure.exc_info[0], failure.exc_info[1], failure.exc_info[2] + + return results + + +def main(args=sys.argv[1:]): + + # parse command line options + parser = OptionParser(description=__doc__) + parser.add_option('--raise', dest='raise_on_error', + default=False, action='store_true', + help="raise on first error") + parser.add_option('--report-first', dest='report_first', + default=False, action='store_true', + help="report the first error only (all tests will still run)") + options, args = parser.parse_args(args) + + # run the tests + results = run_tests(**options.__dict__) + + # check for failure + failed = False + for result in results.values(): + if result[0]: # failure count; http://docs.python.org/library/doctest.html#basic-api + failed = True + break + if failed: + sys.exit(1) # error + +if __name__ == '__main__': + main()
new file mode 100644 --- /dev/null +++ b/build/tests/test_expressionparser.txt @@ -0,0 +1,120 @@ +Test Expressionparser +===================== + +Test the conditional expression parser. + +Boilerplate:: + + >>> from manifestparser import parse + +Test basic values:: + + >>> parse("1") + 1 + >>> parse("100") + 100 + >>> parse("true") + True + >>> parse("false") + False + >>> '' == parse('""') + True + >>> parse('"foo bar"') + 'foo bar' + >>> parse("'foo bar'") + 'foo bar' + >>> parse("foo", foo=1) + 1 + >>> parse("bar", bar=True) + True + >>> parse("abc123", abc123="xyz") + 'xyz' + +Test equality:: + + >>> parse("true == true") + True + >>> parse("false == false") + True + >>> parse("false == false") + True + >>> parse("1 == 1") + True + >>> parse("100 == 100") + True + >>> parse('"some text" == "some text"') + True + >>> parse("true != false") + True + >>> parse("1 != 2") + True + >>> parse('"text" != "other text"') + True + >>> parse("foo == true", foo=True) + True + >>> parse("foo == 1", foo=1) + True + >>> parse('foo == "bar"', foo='bar') + True + >>> parse("foo == bar", foo=True, bar=True) + True + >>> parse("true == foo", foo=True) + True + >>> parse("foo != true", foo=False) + True + >>> parse("foo != 2", foo=1) + True + >>> parse('foo != "bar"', foo='abc') + True + >>> parse("foo != bar", foo=True, bar=False) + True + >>> parse("true != foo", foo=False) + True + >>> parse("!false") + True + +Test conjunctions:: + + >>> parse("true && true") + True + >>> parse("true || false") + True + >>> parse("false || false") + False + >>> parse("true && false") + False + >>> parse("true || false && false") + True + +Test parentheses:: + + >>> parse("(true)") + True + >>> parse("(10)") + 10 + >>> parse('("foo")') + 'foo' + >>> parse("(foo)", foo=1) + 1 + >>> parse("(true == true)") + True + >>> parse("(true != false)") + True + >>> parse("(true && true)") + True + >>> parse("(true || false)") + True + >>> parse("(true && true || false)") + True + >>> parse("(true || false) && false") + False + >>> parse("(true || false) && true") + True + >>> parse("true && (true || false)") + True + >>> parse("true && (true || false)") + True + >>> parse("(true && false) || (true && (true || false))") + True + +
new file mode 100644 --- /dev/null +++ b/build/tests/test_manifestparser.txt @@ -0,0 +1,217 @@ +Test the manifest parser +======================== + +You must have ManifestDestiny installed before running these tests. +Run ``python manifestparser.py setup develop`` with setuptools installed. + +Ensure basic parser is sane:: + + >>> from manifestparser import ManifestParser + >>> parser = ManifestParser() + >>> parser.read('mozmill-example.ini') + >>> tests = parser.tests + >>> len(tests) == len(file('mozmill-example.ini').read().strip().splitlines()) + True + +Ensure that capitalization and order aren't an issue: + + >>> lines = ['[%s]' % test['name'] for test in tests] + >>> lines == file('mozmill-example.ini').read().strip().splitlines() + True + +Show how you select subsets of tests: + + >>> parser.read('mozmill-restart-example.ini') + >>> restart_tests = parser.get(type='restart') + >>> len(restart_tests) < len(parser.tests) + True + >>> import os + >>> len(restart_tests) == len(parser.get(manifest=os.path.abspath('mozmill-restart-example.ini'))) + True + >>> assert not [test for test in restart_tests if test['manifest'] != os.path.abspath('mozmill-restart-example.ini')] + >>> parser.get('name', tags=['foo']) + ['restartTests/testExtensionInstallUninstall/test2.js', 'restartTests/testExtensionInstallUninstall/test1.js'] + >>> parser.get('name', foo='bar') + ['restartTests/testExtensionInstallUninstall/test2.js'] + +Illustrate how include works:: + + >>> parser = ManifestParser(manifests=('include-example.ini',)) + +All of the tests should be included, in order:: + + >>> parser.get('name') + ['crash-handling', 'fleem', 'flowers'] + >>> [(test['name'], os.path.basename(test['manifest'])) for test in parser.tests] + [('crash-handling', 'bar.ini'), ('fleem', 'include-example.ini'), ('flowers', 'foo.ini')] + +The manifests should be there too:: + + >>> len(parser.manifests()) + 3 + +We're already in the root directory:: + + >>> os.getcwd() == parser.rootdir + True + +DEFAULT values should persist across includes, unless they're +overwritten. In this example, include-example.ini sets foo=bar, but +its overridden to fleem in bar.ini:: + + >>> parser.get('name', foo='bar') + ['fleem', 'flowers'] + >>> parser.get('name', foo='fleem') + ['crash-handling'] + +Passing parameters in the include section allows defining variables in +the submodule scope: + + >>> parser.get('name', tags=['red']) + ['flowers'] + +However, this should be overridable from the DEFAULT section in the +included file and that overridable via the key directly connected to +the test:: + + >>> parser.get(name='flowers')[0]['blue'] + 'ocean' + >>> parser.get(name='flowers')[0]['yellow'] + 'submarine' + +You can query multiple times if you need to:: + + >>> flowers = parser.get(foo='bar') + >>> len(flowers) + 2 + >>> roses = parser.get(tests=flowers, red='roses') + +Using the inverse flag should invert the set of tests returned:: + + >>> parser.get('name', inverse=True, tags=['red']) + ['crash-handling', 'fleem'] + +All of the included tests actually exist:: + + >>> [i['name'] for i in parser.missing()] + [] + +Write the output to a manifest: + + >>> from StringIO import StringIO + >>> buffer = StringIO() + >>> parser.write(fp=buffer, global_kwargs={'foo': 'bar'}) + >>> buffer.getvalue().strip() + '[DEFAULT]\nfoo = bar\n\n[fleem]\n\n[include/flowers]\nblue = ocean\nred = roses\nyellow = submarine' + +Test our ability to convert a static directory structure to a +manifest. First, stub out a directory with files in it:: + + >>> import shutil, tempfile + >>> def create_stub(): + ... directory = tempfile.mkdtemp() + ... for i in 'foo', 'bar', 'fleem': + ... file(os.path.join(directory, i), 'w').write(i) + ... subdir = os.path.join(directory, 'subdir') + ... os.mkdir(subdir) + ... file(os.path.join(subdir, 'subfile'), 'w').write('baz') + ... return directory + >>> stub = create_stub() + >>> os.path.exists(stub) and os.path.isdir(stub) + True + +Make a manifest for it:: + + >>> from manifestparser import convert + >>> print convert([stub]) + [bar] + [fleem] + [foo] + [subdir/subfile] + >>> shutil.rmtree(stub) + +Now do the same thing but keep the manifests in place:: + + >>> stub = create_stub() + >>> convert([stub], write='manifest.ini') + >>> sorted(os.listdir(stub)) + ['bar', 'fleem', 'foo', 'manifest.ini', 'subdir'] + >>> parser = ManifestParser() + >>> parser.read(os.path.join(stub, 'manifest.ini')) + >>> [i['name'] for i in parser.tests] + ['subfile', 'bar', 'fleem', 'foo'] + >>> parser = ManifestParser() + >>> parser.read(os.path.join(stub, 'subdir', 'manifest.ini')) + >>> len(parser.tests) + 1 + >>> parser.tests[0]['name'] + 'subfile' + >>> shutil.rmtree(stub) + +Test our ability to copy a set of manifests:: + + >>> tempdir = tempfile.mkdtemp() + >>> manifest = ManifestParser(manifests=('include-example.ini',)) + >>> manifest.copy(tempdir) + >>> sorted(os.listdir(tempdir)) + ['fleem', 'include', 'include-example.ini'] + >>> sorted(os.listdir(os.path.join(tempdir, 'include'))) + ['bar.ini', 'crash-handling', 'flowers', 'foo.ini'] + >>> from_manifest = ManifestParser(manifests=('include-example.ini',)) + >>> to_manifest = os.path.join(tempdir, 'include-example.ini') + >>> to_manifest = ManifestParser(manifests=(to_manifest,)) + >>> to_manifest.get('name') == from_manifest.get('name') + True + >>> shutil.rmtree(tempdir) + +Test our ability to update tests from a manifest and a directory of +files:: + + >>> tempdir = tempfile.mkdtemp() + >>> for i in range(10): + ... file(os.path.join(tempdir, str(i)), 'w').write(str(i)) + +First, make a manifest:: + + >>> manifest = convert([tempdir]) + >>> newtempdir = tempfile.mkdtemp() + >>> manifest_file = os.path.join(newtempdir, 'manifest.ini') + >>> file(manifest_file,'w').write(manifest) + >>> manifest = ManifestParser(manifests=(manifest_file,)) + >>> manifest.get('name') == [str(i) for i in range(10)] + True + +All of the tests are initially missing:: + + >>> [i['name'] for i in manifest.missing()] == [str(i) for i in range(10)] + True + +But then we copy one over:: + + >>> manifest.get('name', name='1') + ['1'] + >>> manifest.update(tempdir, name='1') + >>> sorted(os.listdir(newtempdir)) + ['1', 'manifest.ini'] + +Update that one file and copy all the "tests":: + + >>> file(os.path.join(tempdir, '1'), 'w').write('secret door') + >>> manifest.update(tempdir) + >>> sorted(os.listdir(newtempdir)) + ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'manifest.ini'] + >>> file(os.path.join(newtempdir, '1')).read().strip() + 'secret door' + +Clean up:: + + >>> shutil.rmtree(tempdir) + >>> shutil.rmtree(newtempdir) + +You can override the path in the section too. This shows that you can +use a relative path:: + + >>> manifest = ManifestParser(manifests=('path-example.ini',)) + >>> manifest.tests[0]['path'] == os.path.abspath('fleem') + True +
new file mode 100644 --- /dev/null +++ b/build/tests/test_testmanifest.txt @@ -0,0 +1,32 @@ +Test the Test Manifest +====================== + +Boilerplate:: + + >>> import os + +Test filtering based on platform:: + + >>> from manifestparser import TestManifest + >>> manifest = TestManifest(manifests=('filter-example.ini',)) + >>> [i['name'] for i in manifest.active_tests(os='win', disabled=False, exists=False)] + ['windowstest', 'fleem'] + >>> [i['name'] for i in manifest.active_tests(os='linux', disabled=False, exists=False)] + ['fleem', 'linuxtest'] + +Look for existing tests. There is only one:: + + >>> [i['name'] for i in manifest.active_tests()] + ['fleem'] + +You should be able to expect failures:: + + >>> last_test = manifest.active_tests(exists=False, toolkit='gtk2')[-1] + >>> last_test['name'] + 'linuxtest' + >>> last_test['expected'] + 'pass' + >>> last_test = manifest.active_tests(exists=False, toolkit='cocoa')[-1] + >>> last_test['expected'] + 'fail' +
--- a/config/pythonpath.py +++ b/config/pythonpath.py @@ -32,9 +32,11 @@ while True: continue break sys.argv.pop(0) script = sys.argv[0] sys.path[0:0] = [os.path.dirname(script)] + paths -execfile(script, {'__name__': '__main__', '__file__': script}) +__name__ = '__main__' +__file__ = script +execfile(script)
--- a/config/rules.mk +++ b/config/rules.mk @@ -151,40 +151,43 @@ testxpcsrcdir = $(topsrcdir)/testing/xpc # Execute all tests in the $(XPCSHELL_TESTS) directories. # See also testsuite-targets.mk 'xpcshell-tests' target for global execution. xpcshell-tests: $(PYTHON) -u $(topsrcdir)/config/pythonpath.py \ -I$(topsrcdir)/build \ $(testxpcsrcdir)/runxpcshelltests.py \ --symbols-path=$(DIST)/crashreporter-symbols \ + --build-info-json=$(DEPTH)/mozinfo.json \ $(EXTRA_TEST_ARGS) \ $(LIBXUL_DIST)/bin/xpcshell \ $(foreach dir,$(XPCSHELL_TESTS),$(testxpcobjdir)/$(relativesrcdir)/$(dir)) # Execute a single test, specified in $(SOLO_FILE), but don't automatically # start the test. Instead, present the xpcshell prompt so the user can # attach a debugger and then start the test. check-interactive: $(PYTHON) -u $(topsrcdir)/config/pythonpath.py \ -I$(topsrcdir)/build \ $(testxpcsrcdir)/runxpcshelltests.py \ --symbols-path=$(DIST)/crashreporter-symbols \ + --build-info-json=$(DEPTH)/mozinfo.json \ --test-path=$(SOLO_FILE) \ --profile-name=$(MOZ_APP_NAME) \ --interactive \ $(LIBXUL_DIST)/bin/xpcshell \ $(foreach dir,$(XPCSHELL_TESTS),$(testxpcobjdir)/$(relativesrcdir)/$(dir)) # Execute a single test, specified in $(SOLO_FILE) check-one: $(PYTHON) -u $(topsrcdir)/config/pythonpath.py \ -I$(topsrcdir)/build \ $(testxpcsrcdir)/runxpcshelltests.py \ --symbols-path=$(DIST)/crashreporter-symbols \ + --build-info-json=$(DEPTH)/mozinfo.json \ --test-path=$(SOLO_FILE) \ --profile-name=$(MOZ_APP_NAME) \ --verbose \ $(EXTRA_TEST_ARGS) \ $(LIBXUL_DIST)/bin/xpcshell \ $(foreach dir,$(XPCSHELL_TESTS),$(testxpcobjdir)/$(relativesrcdir)/$(dir)) endif # XPCSHELL_TESTS
--- a/testing/testsuite-targets.mk +++ b/testing/testsuite-targets.mk @@ -161,16 +161,17 @@ GARBAGE += $(addsuffix .log,$(MOCHITESTS # Execute all xpcshell tests in the directories listed in the manifest. # See also config/rules.mk 'xpcshell-tests' target for local execution. # Usage: |make [TEST_PATH=...] [EXTRA_TEST_ARGS=...] xpcshell-tests|. xpcshell-tests: $(PYTHON) -u $(topsrcdir)/config/pythonpath.py \ -I$(topsrcdir)/build \ $(topsrcdir)/testing/xpcshell/runxpcshelltests.py \ --manifest=$(DEPTH)/_tests/xpcshell/xpcshell.ini \ + --build-info-json=$(DEPTH)/mozinfo.json \ --no-logfiles \ $(SYMBOLS_PATH) \ $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS) \ $(LIBXUL_DIST)/bin/xpcshell # install and run the mozmill tests $(DEPTH)/_tests/mozmill: $(MAKE) -C $(DEPTH)/testing/mozmill install-develop PKG_STAGE=../../_tests
--- a/testing/xpcshell/Makefile.in +++ b/testing/xpcshell/Makefile.in @@ -59,16 +59,17 @@ TEST_HARNESS_FILES := \ remotexpcshelltests.py \ head.js \ $(NULL) # Extra files needed from $(topsrcdir)/build EXTRA_BUILD_FILES := \ automationutils.py \ manifestparser.py \ + mozinfo.py \ poster.zip \ $(NULL) # And files for running xpcshell remotely from $(topsrcdir)/build/mobile MOBILE_BUILD_FILES := \ devicemanager.py \ $(NULL) @@ -81,15 +82,21 @@ TEST_HARNESS_COMPONENTS := \ # Rules for staging the necessary harness bits for a test package PKG_STAGE = $(DIST)/test-package-stage libs:: $(INSTALL) $(srcdir)/xpcshell.ini $(DEPTH)/_tests/xpcshell cp $(srcdir)/xpcshell.ini $(DEPTH)/_tests/xpcshell/all-test-dirs.list +# Run selftests +check:: + OBJDIR=$(DEPTH) $(PYTHON) $(topsrcdir)/config/pythonpath.py \ + -I$(topsrcdir)/build $(srcdir)/selftest.py + stage-package: $(NSINSTALL) -D $(PKG_STAGE)/xpcshell/tests @(cd $(srcdir) && tar $(TAR_CREATE_FLAGS) - $(TEST_HARNESS_FILES)) | (cd $(PKG_STAGE)/xpcshell && tar -xf -) @(cd $(topsrcdir)/build && tar $(TAR_CREATE_FLAGS) - $(EXTRA_BUILD_FILES)) | (cd $(PKG_STAGE)/xpcshell && tar -xf -) + @cp $(DEPTH)/mozinfo.json $(PKG_STAGE)/xpcshell @(cd $(topsrcdir)/build/mobile && tar $(TAR_CREATE_FLAGS) - $(MOBILE_BUILD_FILES)) | (cd $(PKG_STAGE)/xpcshell && tar -xf -) @(cd $(DEPTH)/_tests/xpcshell/ && tar $(TAR_CREATE_FLAGS) - *) | (cd $(PKG_STAGE)/xpcshell/tests && tar -xf -) @(cd $(DIST)/bin/components && tar $(TAR_CREATE_FLAGS) - $(TEST_HARNESS_COMPONENTS)) | (cd $(PKG_STAGE)/bin/components && tar -xf -)
new file mode 100644 --- /dev/null +++ b/testing/xpcshell/example/unit/test_fail.js @@ -0,0 +1,4 @@ +function run_test() { + // This test expects to fail. + do_check_true(false); +}
new file mode 100644 --- /dev/null +++ b/testing/xpcshell/example/unit/test_skip.js @@ -0,0 +1,4 @@ +function run_test() { + // This test expects to fail. + do_check_true(false); +}
--- a/testing/xpcshell/example/unit/xpcshell.ini +++ b/testing/xpcshell/example/unit/xpcshell.ini @@ -6,8 +6,14 @@ tail = [test_get_file.js] [test_get_idle.js] [test_import_module.js] [test_load.js] [test_load_httpd_js.js] [test_location.js] [test_profile.js] [test_sample.js] + +[test_fail.js] +fail-if = true + +[test_skip.js] +skip-if = true
--- a/testing/xpcshell/runxpcshelltests.py +++ b/testing/xpcshell/runxpcshelltests.py @@ -39,53 +39,61 @@ # ***** END LICENSE BLOCK ***** */ import re, sys, os, os.path, logging, shutil, signal, math from glob import glob from optparse import OptionParser from subprocess import Popen, PIPE, STDOUT from tempfile import mkdtemp, gettempdir import manifestparser +import mozinfo from automationutils import * +#TODO: replace this with json.loads when Python 2.6 is required. +def parse_json(j): + """ + Awful hack to parse a restricted subset of JSON strings into Python dicts. + """ + return eval(j, {'true':True,'false':False,'null':None}) + """ Control-C handling """ gotSIGINT = False def markGotSIGINT(signum, stackFrame): global gotSIGINT gotSIGINT = True class XPCShellTests(object): log = logging.getLogger() oldcwd = os.getcwd() - def __init__(self): + def __init__(self, log=sys.stdout): """ Init logging """ - handler = logging.StreamHandler(sys.stdout) + handler = logging.StreamHandler(log) self.log.setLevel(logging.INFO) self.log.addHandler(handler) def buildTestList(self): """ read the xpcshell.ini manifest and set self.alltests to be an array of test objects. if we are chunking tests, it will be done here as well """ - mp = manifestparser.ManifestParser(strict=False) + mp = manifestparser.TestManifest(strict=False) if self.manifest is None: for testdir in self.testdirs: if testdir: mp.read(os.path.join(testdir, 'xpcshell.ini')) else: mp.read(self.manifest) self.buildTestPath() - self.alltests = mp.tests + self.alltests = mp.active_tests(**mozinfo.info) if self.singleFile is None and self.totalChunks > 1: self.chunkTests() def chunkTests(self): """ Split the list of tests up into [totalChunks] pieces and filter the self.alltests based on thisChunk, so we only run a subset. @@ -133,17 +141,20 @@ class XPCShellTests(object): self.httpdManifest = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.manifest') self.httpdManifest = replaceBackSlashes(self.httpdManifest) if self.xrePath is None: self.xrePath = os.path.dirname(self.xpcshell) else: self.xrePath = os.path.abspath(self.xrePath) - + + if self.mozInfo is None: + self.mozInfo = os.path.join(self.testharnessdir, "mozinfo.json") + def buildEnvironment(self): """ Create and returns a dictionary of self.env to include all the appropriate env variables and values. On a remote system, we overload this to set different values and are missing things like os.environ and PATH. """ self.env = dict(os.environ) # Make assertions fatal self.env["XPCOM_DEBUG_BREAK"] = "stack-and-abort" @@ -276,17 +287,17 @@ class XPCShellTests(object): self.removeDir(profileDir) except: pass os.makedirs(profileDir) else: profileDir = mkdtemp() self.env["XPCSHELL_TEST_PROFILE_DIR"] = profileDir if self.interactive or self.singleFile: - print "TEST-INFO | profile dir is %s" % profileDir + self.log.info("TEST-INFO | profile dir is %s" % profileDir) return profileDir def setupLeakLogging(self): """ Enable leaks (only) detection to its own log file and set environment variables. On a remote system, we overload this to use a remote filename and path structure """ @@ -368,17 +379,17 @@ class XPCShellTests(object): '-e', 'const _HEAD_FILES = [%s];' % cmdH, '-e', 'const _TAIL_FILES = [%s];' % cmdT] def runTests(self, xpcshell, xrePath=None, symbolsPath=None, manifest=None, testdirs=[], testPath=None, interactive=False, verbose=False, keepGoing=False, logfiles=True, thisChunk=1, totalChunks=1, debugger=None, debuggerArgs=None, debuggerInteractive=False, - profileName=None): + profileName=None, mozInfo=None): """Run xpcshell tests. |xpcshell|, is the xpcshell executable to use to run the tests. |xrePath|, if provided, is the path to the XRE to use. |symbolsPath|, if provided is the path to a directory containing breakpad symbols for processing crashes in tests. |manifest|, if provided, is a file containing a list of test directories to run. @@ -389,17 +400,18 @@ class XPCShellTests(object): instead of automatically executing the test. |verbose|, if set to True, will cause stdout/stderr from tests to be printed always |logfiles|, if set to False, indicates not to save output to log files. Non-interactive only option. |debuggerInfo|, if set, specifies the debugger and debugger arguments that will be used to launch xpcshell. |profileName|, if set, specifies the name of the application for the profile - directory if running only a subset of tests + directory if running only a subset of tests. + |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict. """ global gotSIGINT self.xpcshell = xpcshell self.xrePath = xrePath self.symbolsPath = symbolsPath self.manifest = manifest @@ -408,60 +420,83 @@ class XPCShellTests(object): self.interactive = interactive self.verbose = verbose self.keepGoing = keepGoing self.logfiles = logfiles self.totalChunks = totalChunks self.thisChunk = thisChunk self.debuggerInfo = getDebuggerInfo(self.oldcwd, debugger, debuggerArgs, debuggerInteractive) self.profileName = profileName or "xpcshell" + self.mozInfo = mozInfo # If we have an interactive debugger, disable ctrl-c. if self.debuggerInfo and self.debuggerInfo["interactive"]: signal.signal(signal.SIGINT, lambda signum, frame: None) if not testdirs and not manifest: # nothing to test! - print >>sys.stderr, "Error: No test dirs or test manifest specified!" + self.log.error("Error: No test dirs or test manifest specified!") return False - passCount = 0 - failCount = 0 + self.testCount = 0 + self.passCount = 0 + self.failCount = 0 + self.todoCount = 0 self.setAbsPath() self.buildXpcsRunArgs() self.buildEnvironment() + + # Handle filenames in mozInfo + if not isinstance(self.mozInfo, dict): + mozInfoFile = self.mozInfo + if not os.path.isfile(mozInfoFile): + self.log.error("Error: couldn't find mozinfo.json at '%s'. Perhaps you need to use --build-info-json?" % mozInfoFile) + return False + self.mozInfo = parse_json(open(mozInfoFile).read()) + mozinfo.update(self.mozInfo) + pStdout, pStderr = self.getPipes() self.buildTestList() for test in self.alltests: name = test['path'] if self.singleFile and not name.endswith(self.singleFile): continue if self.testPath and name.find(self.testPath) == -1: continue + self.testCount += 1 + + # Check for skipped tests + if 'disabled' in test: + self.log.info("TEST-INFO | skipping %s | %s" % + (name, test['disabled'])) + continue + # Check for known-fail tests + expected = test['expected'] == 'pass' + testdir = os.path.dirname(name) self.buildXpcsCmd(testdir) testHeadFiles = self.getHeadFiles(test) testTailFiles = self.getTailFiles(test) cmdH = self.buildCmdHead(testHeadFiles, testTailFiles, self.xpcsCmd) # create a temp dir that the JS harness can stick a profile in self.profileDir = self.setupProfileDir() self.leakLogFile = self.setupLeakLogging() # The test file will have to be loaded after the head files. cmdT = ['-e', 'const _TEST_FILE = ["%s"];' % replaceBackSlashes(name)] try: - print "TEST-INFO | %s | running test ..." % name + self.log.info("TEST-INFO | %s | running test ..." % name) proc = self.launchProcess(cmdH + cmdT + self.xpcsRunArgs, stdout=pStdout, stderr=pStderr, env=self.env, cwd=testdir) # Allow user to kill hung subprocess with SIGINT w/o killing this script # - don't move this line above launchProcess, or child will inherit the SIG_IGN signal.signal(signal.SIGINT, markGotSIGINT) # |stderr == None| as |pStderr| was either |None| or redirected to |stdout|. @@ -469,32 +504,39 @@ class XPCShellTests(object): signal.signal(signal.SIGINT, signal.SIG_DFL) if interactive: # Not sure what else to do here... return True def print_stdout(stdout): """Print stdout line-by-line to avoid overflowing buffers.""" - print ">>>>>>>" + self.log.info(">>>>>>>") for line in stdout.splitlines(): - print line - print "<<<<<<<" + self.log.info(line) + self.log.info("<<<<<<<") - if (self.getReturnCode(proc) != 0) or \ - (stdout and re.search("^((parent|child): )?TEST-UNEXPECTED-", stdout, re.MULTILINE)) or \ - (stdout and re.search(": SyntaxError:", stdout, re.MULTILINE)): - print "TEST-UNEXPECTED-FAIL | %s | test failed (with xpcshell return code: %d), see following log:" % (name, self.getReturnCode(proc)) + result = not ((self.getReturnCode(proc) != 0) or + (stdout and re.search("^((parent|child): )?TEST-UNEXPECTED-", + stdout, re.MULTILINE)) or + (stdout and re.search(": SyntaxError:", stdout, + re.MULTILINE))) + + if result != expected: + self.log.error("TEST-UNEXPECTED-%s | %s | test failed (with xpcshell return code: %d), see following log:" % ("FAIL" if expected else "PASS", name, self.getReturnCode(proc))) print_stdout(stdout) - failCount += 1 + self.failCount += 1 else: - print "TEST-PASS | %s | test passed" % name + self.log.info("TEST-%s | %s | test passed" % ("PASS" if expected else "KNOWN-FAIL", name)) if verbose: print_stdout(stdout) - passCount += 1 + if expected: + self.passCount += 1 + else: + self.todoCount += 1 checkForCrashes(testdir, self.symbolsPath, testName=name) # Find child process(es) leak log(s), if any: See InitLog() in # xpcom/base/nsTraceRefcntImpl.cpp for logfile naming logic leakLogs = [self.leakLogFile] for childLog in glob(os.path.join(self.profileDir, "runxpcshelltests_leaks_*_pid*.log")): if os.path.isfile(childLog): leakLogs += [childLog] @@ -504,34 +546,35 @@ class XPCShellTests(object): if self.logfiles and stdout: self.createLogFile(name, stdout, leakLogs) finally: # We don't want to delete the profile when running check-interactive # or check-one. if self.profileDir and not self.interactive and not self.singleFile: self.removeDir(self.profileDir) if gotSIGINT: - print "TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution" + self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution") if (keepGoing): gotSIGINT = False else: break - if passCount == 0 and failCount == 0: - print "TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?" - failCount = 1 + if self.testCount == 0: + self.log.error("TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?") + self.failCount = 1 - print """INFO | Result summary: + self.log.info("""INFO | Result summary: INFO | Passed: %d -INFO | Failed: %d""" % (passCount, failCount) +INFO | Failed: %d +INFO | Todo: %d""" % (self.passCount, self.failCount, self.todoCount)) if gotSIGINT and not keepGoing: - print "TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \ - "(Use --keep-going to keep running tests after killing one with SIGINT)" + log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \ + "(Use --keep-going to keep running tests after killing one with SIGINT)") return False - return failCount == 0 + return self.failCount == 0 class XPCShellOptions(OptionParser): def __init__(self): """Process command line arguments and call runTests() to do the real work.""" OptionParser.__init__(self) addCommonOptions(self) self.add_option("--interactive", @@ -559,16 +602,19 @@ class XPCShellOptions(OptionParser): type = "int", dest = "totalChunks", default=1, help = "how many chunks to split the tests up into") self.add_option("--this-chunk", type = "int", dest = "thisChunk", default=1, help = "which chunk to run between 1 and --total-chunks") self.add_option("--profile-name", type = "string", dest="profileName", default=None, help="name of application profile being tested") + self.add_option("--build-info-json", + type = "string", dest="mozInfo", default=None, + help="path to a mozinfo.json including information about the build configuration. defaults to looking for mozinfo.json next to the script.") def main(): parser = XPCShellOptions() options, args = parser.parse_args() if len(args) < 2 and options.manifest is None or \ (len(args) < 1 and options.manifest is not None): print >>sys.stderr, """Usage: %s <path to xpcshell> <test dirs>
new file mode 100644 --- /dev/null +++ b/testing/xpcshell/selftest.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ +# + +import sys, os, unittest, tempfile, shutil +from StringIO import StringIO + +from runxpcshelltests import XPCShellTests + +objdir = os.path.abspath(os.environ["OBJDIR"]) +xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell") +if sys.platform == "win32": + xpcshellBin += ".exe" + +SIMPLE_PASSING_TEST = "function run_test() { do_check_true(true); }" +SIMPLE_FAILING_TEST = "function run_test() { do_check_true(false); }" + +class XPCShellTestsTests(unittest.TestCase): + """ + Yes, these are unit tests for a unit test harness. + """ + def setUp(self): + self.log = StringIO() + self.tempdir = tempfile.mkdtemp() + self.x = XPCShellTests(log=self.log) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def writeFile(self, name, contents): + """ + Write |contents| to a file named |name| in the temp directory, + and return the full path to the file. + """ + fullpath = os.path.join(self.tempdir, name) + with open(fullpath, "w") as f: + f.write(contents) + return fullpath + + def writeManifest(self, tests): + """ + Write an xpcshell.ini in the temp directory and set + self.manifest to its pathname. |tests| is a list containing + either strings (for test names), or tuples with a test name + as the first element and manifest conditions as the following + elements. + """ + testlines = [] + for t in tests: + testlines.append("[%s]" % (t if isinstance(t, basestring) + else t[0])) + if isinstance(t, tuple): + testlines.extend(t[1:]) + self.manifest = self.writeFile("xpcshell.ini", """ +[DEFAULT] +head = +tail = + +""" + "\n".join(testlines)) + + def assertTestResult(self, expected, mozInfo={}): + """ + Assert that self.x.runTests with manifest=self.manifest + returns |expected|. + """ + self.assertEquals(expected, + self.x.runTests(xpcshellBin, + manifest=self.manifest, + mozInfo=mozInfo), + msg="""Tests should have %s, log: +======== +%s +======== +""" % ("passed" if expected else "failed", self.log.getvalue())) + + def _assertLog(self, s, expected): + l = self.log.getvalue() + self.assertEqual(expected, s in l, + msg="""Value %s %s in log: +======== +%s +========""" % (s, "expected" if expected else "not expected", l)) + + def assertInLog(self, s): + """ + Assert that the string |s| is contained in self.log. + """ + self._assertLog(s, True) + + def assertNotInLog(self, s): + """ + Assert that the string |s| is not contained in self.log. + """ + self._assertLog(s, False) + + def testPass(self): + """ + Check that a simple test without any manifest conditions passes. + """ + self.writeFile("test_basic.js", SIMPLE_PASSING_TEST) + self.writeManifest(["test_basic.js"]) + + self.assertTestResult(True) + self.assertEquals(1, self.x.testCount) + self.assertEquals(1, self.x.passCount) + self.assertEquals(0, self.x.failCount) + self.assertEquals(0, self.x.todoCount) + self.assertInLog("TEST-PASS") + self.assertNotInLog("TEST-UNEXPECTED-FAIL") + + def testFail(self): + """ + Check that a simple failing test without any manifest conditions fails. + """ + self.writeFile("test_basic.js", SIMPLE_FAILING_TEST) + self.writeManifest(["test_basic.js"]) + + self.assertTestResult(False) + self.assertEquals(1, self.x.testCount) + self.assertEquals(0, self.x.passCount) + self.assertEquals(1, self.x.failCount) + self.assertEquals(0, self.x.todoCount) + self.assertInLog("TEST-UNEXPECTED-FAIL") + self.assertNotInLog("TEST-PASS") + + def testPassFail(self): + """ + Check that running more than one test works. + """ + self.writeFile("test_pass.js", SIMPLE_PASSING_TEST) + self.writeFile("test_fail.js", SIMPLE_FAILING_TEST) + self.writeManifest(["test_pass.js", "test_fail.js"]) + + self.assertTestResult(False) + self.assertEquals(2, self.x.testCount) + self.assertEquals(1, self.x.passCount) + self.assertEquals(1, self.x.failCount) + self.assertEquals(0, self.x.todoCount) + self.assertInLog("TEST-PASS") + self.assertInLog("TEST-UNEXPECTED-FAIL") + + def testSkip(self): + """ + Check that a simple failing test skipped in the manifest does + not cause failure. + """ + self.writeFile("test_basic.js", SIMPLE_FAILING_TEST) + self.writeManifest([("test_basic.js", "skip-if = true")]) + self.assertTestResult(True) + self.assertEquals(1, self.x.testCount) + self.assertEquals(0, self.x.passCount) + self.assertEquals(0, self.x.failCount) + self.assertEquals(0, self.x.todoCount) + self.assertNotInLog("TEST-UNEXPECTED-FAIL") + self.assertNotInLog("TEST-PASS") + + def testKnownFail(self): + """ + Check that a simple failing test marked as known-fail in the manifest + does not cause failure. + """ + self.writeFile("test_basic.js", SIMPLE_FAILING_TEST) + self.writeManifest([("test_basic.js", "fail-if = true")]) + self.assertTestResult(True) + self.assertEquals(1, self.x.testCount) + self.assertEquals(0, self.x.passCount) + self.assertEquals(0, self.x.failCount) + self.assertEquals(1, self.x.todoCount) + self.assertInLog("TEST-KNOWN-FAIL") + # This should be suppressed because the harness doesn't include + # the full log from the xpcshell run when things pass. + self.assertNotInLog("TEST-UNEXPECTED-FAIL") + self.assertNotInLog("TEST-PASS") + + def testUnexpectedPass(self): + """ + Check that a simple failing test marked as known-fail in the manifest + that passes causes an unexpected pass. + """ + self.writeFile("test_basic.js", SIMPLE_PASSING_TEST) + self.writeManifest([("test_basic.js", "fail-if = true")]) + self.assertTestResult(False) + self.assertEquals(1, self.x.testCount) + self.assertEquals(0, self.x.passCount) + self.assertEquals(1, self.x.failCount) + self.assertEquals(0, self.x.todoCount) + # From the outer (Python) harness + self.assertInLog("TEST-UNEXPECTED-PASS") + self.assertNotInLog("TEST-KNOWN-FAIL") + # From the inner (JS) harness + self.assertInLog("TEST-PASS") + + def testReturnNonzero(self): + """ + Check that a test where xpcshell returns nonzero fails. + """ + self.writeFile("test_error.js", "throw 'foo'") + self.writeManifest(["test_error.js"]) + + self.assertTestResult(False) + self.assertEquals(1, self.x.testCount) + self.assertEquals(0, self.x.passCount) + self.assertEquals(1, self.x.failCount) + self.assertEquals(0, self.x.todoCount) + self.assertInLog("TEST-UNEXPECTED-FAIL") + self.assertNotInLog("TEST-PASS") + +if __name__ == "__main__": + unittest.main()