author | Gregory Szorc <gps@mozilla.com> |
Mon, 27 Feb 2012 19:53:00 -0800 | |
changeset 87899 | d913e27b169bf432b3f7d31ddec50903208c885c |
parent 87898 | dbe3f8bad3b5ecb2d10719f83b9956c172a50bde |
child 87900 | 32397978a3424fac2c6987cd6d111ebae69da8f3 |
push id | 22160 |
push user | mbrubeck@mozilla.com |
push date | Tue, 28 Feb 2012 17:21:33 +0000 |
treeherder | mozilla-central@dde4e0089a18 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | Waldo |
bugs | 725478 |
milestone | 13.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
testing/xpcshell/runxpcshelltests.py | file | annotate | diff | comparison | revisions | |
testing/xpcshell/selftest.py | file | annotate | diff | comparison | revisions |
--- a/testing/xpcshell/runxpcshelltests.py +++ b/testing/xpcshell/runxpcshelltests.py @@ -34,16 +34,17 @@ # decision by deleting the provisions above and replace them with the notice # and other provisions required by the GPL or the LGPL. If you do not delete # the provisions above, a recipient may use your version of this file under # the terms of any one of the MPL, the GPL or the LGPL. # # ***** END LICENSE BLOCK ***** */ import re, sys, os, os.path, logging, shutil, signal, math, time +import xml.dom.minidom from glob import glob from optparse import OptionParser from subprocess import Popen, PIPE, STDOUT from tempfile import mkdtemp, gettempdir import manifestparser import mozinfo import random @@ -382,22 +383,127 @@ class XPCShellTests(object): def buildCmdTestFile(self, name): """ Build the command line arguments for the test file. On a remote system, this may be overloaded to use a remote path structure. """ return ['-e', 'const _TEST_FILE = ["%s"];' % replaceBackSlashes(name)] + def writeXunitResults(self, results, name=None, filename=None, fh=None): + """ + Write Xunit XML from results. + + The function receives an iterable of results dicts. Each dict must have + the following keys: + + classname - The "class" name of the test. + name - The simple name of the test. + + In addition, it must have one of the following saying how the test + executed: + + passed - Boolean indicating whether the test passed. False if it + failed. + skipped - True if the test was skipped. + + The following keys are optional: + + time - Execution time of the test in decimal seconds. + failure - Dict describing test failure. Requires keys: + type - String type of failure. + message - String describing basic failure. + text - Verbose string describing failure. + + Arguments: + + |name|, Name of the test suite. Many tools expect Java class dot notation + e.g. dom.simple.foo. A directory with '/' converted to '.' is a good + choice. + |fh|, File handle to write XML to. + |filename|, File name to write XML to. + |results|, Iterable of tuples describing the results. + """ + if filename is None and fh is None: + raise Exception("One of filename or fh must be defined.") + + if name is None: + name = "xpcshell" + else: + assert isinstance(name, str) + + if filename is not None: + fh = open(filename, 'wb') + + doc = xml.dom.minidom.Document() + testsuite = doc.createElement("testsuite") + testsuite.setAttribute("name", name) + doc.appendChild(testsuite) + + total = 0 + passed = 0 + failed = 0 + skipped = 0 + + for result in results: + total += 1 + + if result.get("skipped", None): + skipped += 1 + elif result["passed"]: + passed += 1 + else: + failed += 1 + + testcase = doc.createElement("testcase") + testcase.setAttribute("classname", result["classname"]) + testcase.setAttribute("name", result["name"]) + + if "time" in result: + testcase.setAttribute("time", str(result["time"])) + else: + # It appears most tools expect the time attribute to be present. + testcase.setAttribute("time", "0") + + if "failure" in result: + failure = doc.createElement("failure") + failure.setAttribute("type", str(result["failure"]["type"])) + failure.setAttribute("message", result["failure"]["message"]) + + # Lossy translation but required to not break CDATA. Also, text could + # be None and Python 2.5's minidom doesn't accept None. Later versions + # do, however. + cdata = result["failure"]["text"] + if not isinstance(cdata, str): + cdata = "" + + cdata = cdata.replace("]]>", "]] >") + text = doc.createCDATASection(cdata) + failure.appendChild(text) + testcase.appendChild(failure) + + if result.get("skipped", None): + e = doc.createElement("skipped") + testcase.appendChild(e) + + testsuite.appendChild(testcase) + + testsuite.setAttribute("tests", str(total)) + testsuite.setAttribute("failures", str(failed)) + testsuite.setAttribute("skip", str(skipped)) + + doc.writexml(fh, addindent=" ", newl="\n", encoding="utf-8") + def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None, - manifest=None, testdirs=[], testPath=None, + manifest=None, testdirs=None, testPath=None, interactive=False, verbose=False, keepGoing=False, logfiles=True, thisChunk=1, totalChunks=1, debugger=None, debuggerArgs=None, debuggerInteractive=False, - profileName=None, mozInfo=None, shuffle=False, **otherOptions): + profileName=None, mozInfo=None, shuffle=False, + xunitFilename=None, xunitName=None, **otherOptions): """Run xpcshell tests. |xpcshell|, is the xpcshell executable to use to run the tests. |xrePath|, if provided, is the path to the XRE to use. |appPath|, if provided, is the path to an application directory. |symbolsPath|, if provided is the path to a directory containing breakpad symbols for processing crashes in tests. |manifest|, if provided, is a file containing a list of @@ -412,20 +518,27 @@ class XPCShellTests(object): |logfiles|, if set to False, indicates not to save output to log files. Non-interactive only option. |debuggerInfo|, if set, specifies the debugger and debugger arguments that will be used to launch xpcshell. |profileName|, if set, specifies the name of the application for the profile directory if running only a subset of tests. |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict. |shuffle|, if True, execute tests in random order. + |xunitFilename|, if set, specifies the filename to which to write xUnit XML + results. + |xunitName|, if outputting an xUnit XML file, the str value to use for the + testsuite name. |otherOptions| may be present for the convenience of subclasses """ - global gotSIGINT + global gotSIGINT + + if testdirs is None: + testdirs = [] self.xpcshell = xpcshell self.xrePath = xrePath self.appPath = appPath self.symbolsPath = symbolsPath self.manifest = manifest self.testdirs = testdirs self.testPath = testPath @@ -468,31 +581,39 @@ class XPCShellTests(object): pStdout, pStderr = self.getPipes() self.buildTestList() if shuffle: random.shuffle(self.alltests) + xunitResults = [] + for test in self.alltests: name = test['path'] if self.singleFile and not name.endswith(self.singleFile): continue if self.testPath and name.find(self.testPath) == -1: continue self.testCount += 1 + xunitResult = {"classname": "xpcshell", "name": test["name"]} + # Check for skipped tests if 'disabled' in test: self.log.info("TEST-INFO | skipping %s | %s" % (name, test['disabled'])) + + xunitResult["skipped"] = True + xunitResults.append(xunitResult) continue + # Check for known-fail tests expected = test['expected'] == 'pass' testdir = os.path.dirname(name) self.buildXpcsCmd(testdir) testHeadFiles = self.getHeadFiles(test) testTailFiles = self.getTailFiles(test) cmdH = self.buildCmdHead(testHeadFiles, testTailFiles, self.xpcsCmd) @@ -536,24 +657,39 @@ class XPCShellTests(object): result = not ((self.getReturnCode(proc) != 0) or (stdout and re.search("^((parent|child): )?TEST-UNEXPECTED-", stdout, re.MULTILINE)) or (stdout and re.search(": SyntaxError:", stdout, re.MULTILINE))) if result != expected: - self.log.error("TEST-UNEXPECTED-%s | %s | test failed (with xpcshell return code: %d), see following log:" % ("FAIL" if expected else "PASS", name, self.getReturnCode(proc))) + failureType = "TEST-UNEXPECTED-%s" % ("FAIL" if expected else "PASS") + message = "%s | %s | test failed (with xpcshell return code: %d), see following log:" % ( + failureType, name, self.getReturnCode(proc)) + self.log.error(message) print_stdout(stdout) self.failCount += 1 + xunitResult["passed"] = False + + xunitResult["failure"] = { + "type": failureType, + "message": message, + "text": stdout + } else: - timeTaken = (time.time() - startTime) * 1000 + now = time.time() + timeTaken = (now - startTime) * 1000 + xunitResult["time"] = now - startTime self.log.info("TEST-%s | %s | test passed (time: %.3fms)" % ("PASS" if expected else "KNOWN-FAIL", name, timeTaken)) if verbose: print_stdout(stdout) + + xunitResult["passed"] = True + if expected: self.passCount += 1 else: self.todoCount += 1 checkForCrashes(testdir, self.symbolsPath, testName=name) # Find child process(es) leak log(s), if any: See InitLog() in # xpcom/base/nsTraceRefcntImpl.cpp for logfile naming logic @@ -567,34 +703,51 @@ class XPCShellTests(object): if self.logfiles and stdout: self.createLogFile(name, stdout, leakLogs) finally: # We don't want to delete the profile when running check-interactive # or check-one. if self.profileDir and not self.interactive and not self.singleFile: self.removeDir(self.profileDir) if gotSIGINT: + xunitResult["passed"] = False + xunitResult["time"] = "0.0" + xunitResult["failure"] = { + "type": "SIGINT", + "message": "Received SIGINT", + "text": "Received SIGINT (control-C) during test execution." + } + self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution") if (keepGoing): gotSIGINT = False else: + xunitResults.append(xunitResult) break + + xunitResults.append(xunitResult) + if self.testCount == 0: self.log.error("TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?") self.failCount = 1 self.log.info("""INFO | Result summary: INFO | Passed: %d INFO | Failed: %d INFO | Todo: %d""" % (self.passCount, self.failCount, self.todoCount)) + if xunitFilename is not None: + self.writeXunitResults(filename=xunitFilename, results=xunitResults, + name=xunitName) + if gotSIGINT and not keepGoing: self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \ "(Use --keep-going to keep running tests after killing one with SIGINT)") return False + return self.failCount == 0 class XPCShellOptions(OptionParser): def __init__(self): """Process command line arguments and call runTests() to do the real work.""" OptionParser.__init__(self) addCommonOptions(self) @@ -632,16 +785,22 @@ class XPCShellOptions(OptionParser): type = "string", dest="profileName", default=None, help="name of application profile being tested") self.add_option("--build-info-json", type = "string", dest="mozInfo", default=None, help="path to a mozinfo.json including information about the build configuration. defaults to looking for mozinfo.json next to the script.") self.add_option("--shuffle", action="store_true", dest="shuffle", default=False, help="Execute tests in random order") + self.add_option("--xunit-file", dest="xunitFilename", + help="path to file where xUnit results will be written.") + self.add_option("--xunit-suite-name", dest="xunitName", + help="name to record for this xUnit test suite. Many " + "tools expect Java class notation, e.g. " + "dom.basic.foo") def main(): parser = XPCShellOptions() options, args = parser.parse_args() if len(args) < 2 and options.manifest is None or \ (len(args) < 1 and options.manifest is not None): print >>sys.stderr, """Usage: %s <path to xpcshell> <test dirs>
--- a/testing/xpcshell/selftest.py +++ b/testing/xpcshell/selftest.py @@ -2,16 +2,17 @@ # # Any copyright is dedicated to the Public Domain. # http://creativecommons.org/publicdomain/zero/1.0/ # from __future__ import with_statement import sys, os, unittest, tempfile, shutil from StringIO import StringIO +from xml.etree.ElementTree import ElementTree from runxpcshelltests import XPCShellTests objdir = os.path.abspath(os.environ["OBJDIR"]) xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell") if sys.platform == "win32": xpcshellBin += ".exe" @@ -56,26 +57,27 @@ class XPCShellTestsTests(unittest.TestCa testlines.extend(t[1:]) self.manifest = self.writeFile("xpcshell.ini", """ [DEFAULT] head = tail = """ + "\n".join(testlines)) - def assertTestResult(self, expected, mozInfo={}, shuffle=False): + def assertTestResult(self, expected, shuffle=False, xunitFilename=None): """ Assert that self.x.runTests with manifest=self.manifest returns |expected|. """ self.assertEquals(expected, self.x.runTests(xpcshellBin, manifest=self.manifest, - mozInfo=mozInfo, - shuffle=shuffle), + mozInfo={}, + shuffle=shuffle, + xunitFilename=xunitFilename), msg="""Tests should have %s, log: ======== %s ======== """ % ("passed" if expected else "failed", self.log.getvalue())) def _assertLog(self, s, expected): l = self.log.getvalue() @@ -219,10 +221,54 @@ tail = self.writeFile(filename, SIMPLE_PASSING_TEST) manifest.append(filename) self.writeManifest(manifest) self.assertTestResult(True, shuffle=True) self.assertEquals(10, self.x.testCount) self.assertEquals(10, self.x.passCount) + def testXunitOutput(self): + """ + Check that Xunit XML files are written. + """ + self.writeFile("test_00.js", SIMPLE_PASSING_TEST) + self.writeFile("test_01.js", SIMPLE_FAILING_TEST) + self.writeFile("test_02.js", SIMPLE_PASSING_TEST) + + manifest = [ + "test_00.js", + "test_01.js", + ("test_02.js", "skip-if = true") + ] + + self.writeManifest(manifest) + + filename = os.path.join(self.tempdir, "xunit.xml") + + self.assertTestResult(False, xunitFilename=filename) + + self.assertTrue(os.path.exists(filename)) + self.assertTrue(os.path.getsize(filename) > 0) + + tree = ElementTree() + tree.parse(filename) + suite = tree.getroot() + + self.assertTrue(suite is not None) + self.assertEqual(suite.get("tests"), "3") + self.assertEqual(suite.get("failures"), "1") + self.assertEqual(suite.get("skip"), "1") + + testcases = suite.findall("testcase") + self.assertEqual(len(testcases), 3) + + for testcase in testcases: + attributes = testcase.keys() + self.assertTrue("classname" in attributes) + self.assertTrue("name" in attributes) + self.assertTrue("time" in attributes) + + self.assertTrue(testcases[1].find("failure") is not None) + self.assertTrue(testcases[2].find("skipped") is not None) + if __name__ == "__main__": unittest.main()