Bug 725478 - Support for generating xUnit result files from xpcshell tests; r=Waldo
authorGregory Szorc <gps@mozilla.com>
Mon, 27 Feb 2012 19:53:00 -0800
changeset 87899 d913e27b169bf432b3f7d31ddec50903208c885c
parent 87898 dbe3f8bad3b5ecb2d10719f83b9956c172a50bde
child 87900 32397978a3424fac2c6987cd6d111ebae69da8f3
push id22160
push usermbrubeck@mozilla.com
push dateTue, 28 Feb 2012 17:21:33 +0000
treeherdermozilla-central@dde4e0089a18 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersWaldo
bugs725478
milestone13.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 725478 - Support for generating xUnit result files from xpcshell tests; r=Waldo
testing/xpcshell/runxpcshelltests.py
testing/xpcshell/selftest.py
--- a/testing/xpcshell/runxpcshelltests.py
+++ b/testing/xpcshell/runxpcshelltests.py
@@ -34,16 +34,17 @@
 # decision by deleting the provisions above and replace them with the notice
 # and other provisions required by the GPL or the LGPL. If you do not delete
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK ***** */
 
 import re, sys, os, os.path, logging, shutil, signal, math, time
+import xml.dom.minidom
 from glob import glob
 from optparse import OptionParser
 from subprocess import Popen, PIPE, STDOUT
 from tempfile import mkdtemp, gettempdir
 import manifestparser
 import mozinfo
 import random
 
@@ -382,22 +383,127 @@ class XPCShellTests(object):
   def buildCmdTestFile(self, name):
     """
       Build the command line arguments for the test file.
       On a remote system, this may be overloaded to use a remote path structure.
     """
     return ['-e', 'const _TEST_FILE = ["%s"];' %
               replaceBackSlashes(name)]
 
+  def writeXunitResults(self, results, name=None, filename=None, fh=None):
+    """
+      Write Xunit XML from results.
+
+      The function receives an iterable of results dicts. Each dict must have
+      the following keys:
+
+        classname - The "class" name of the test.
+        name - The simple name of the test.
+
+      In addition, it must have one of the following saying how the test
+      executed:
+
+        passed - Boolean indicating whether the test passed. False if it
+          failed.
+        skipped - True if the test was skipped.
+
+      The following keys are optional:
+
+        time - Execution time of the test in decimal seconds.
+        failure - Dict describing test failure. Requires keys:
+          type - String type of failure.
+          message - String describing basic failure.
+          text - Verbose string describing failure.
+
+      Arguments:
+
+      |name|, Name of the test suite. Many tools expect Java class dot notation
+        e.g. dom.simple.foo. A directory with '/' converted to '.' is a good
+        choice.
+      |fh|, File handle to write XML to.
+      |filename|, File name to write XML to.
+      |results|, Iterable of tuples describing the results.
+    """
+    if filename is None and fh is None:
+      raise Exception("One of filename or fh must be defined.")
+
+    if name is None:
+      name = "xpcshell"
+    else:
+      assert isinstance(name, str)
+
+    if filename is not None:
+      fh = open(filename, 'wb')
+
+    doc = xml.dom.minidom.Document()
+    testsuite = doc.createElement("testsuite")
+    testsuite.setAttribute("name", name)
+    doc.appendChild(testsuite)
+
+    total = 0
+    passed = 0
+    failed = 0
+    skipped = 0
+
+    for result in results:
+      total += 1
+
+      if result.get("skipped", None):
+        skipped += 1
+      elif result["passed"]:
+        passed += 1
+      else:
+        failed += 1
+
+      testcase = doc.createElement("testcase")
+      testcase.setAttribute("classname", result["classname"])
+      testcase.setAttribute("name", result["name"])
+
+      if "time" in result:
+        testcase.setAttribute("time", str(result["time"]))
+      else:
+        # It appears most tools expect the time attribute to be present.
+        testcase.setAttribute("time", "0")
+
+      if "failure" in result:
+        failure = doc.createElement("failure")
+        failure.setAttribute("type", str(result["failure"]["type"]))
+        failure.setAttribute("message", result["failure"]["message"])
+
+        # Lossy translation but required to not break CDATA. Also, text could
+        # be None and Python 2.5's minidom doesn't accept None. Later versions
+        # do, however.
+        cdata = result["failure"]["text"]
+        if not isinstance(cdata, str):
+            cdata = ""
+
+        cdata = cdata.replace("]]>", "]] >")
+        text = doc.createCDATASection(cdata)
+        failure.appendChild(text)
+        testcase.appendChild(failure)
+
+      if result.get("skipped", None):
+        e = doc.createElement("skipped")
+        testcase.appendChild(e)
+
+      testsuite.appendChild(testcase)
+
+    testsuite.setAttribute("tests", str(total))
+    testsuite.setAttribute("failures", str(failed))
+    testsuite.setAttribute("skip", str(skipped))
+
+    doc.writexml(fh, addindent="  ", newl="\n", encoding="utf-8")
+
   def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None,
-               manifest=None, testdirs=[], testPath=None,
+               manifest=None, testdirs=None, testPath=None,
                interactive=False, verbose=False, keepGoing=False, logfiles=True,
                thisChunk=1, totalChunks=1, debugger=None,
                debuggerArgs=None, debuggerInteractive=False,
-               profileName=None, mozInfo=None, shuffle=False, **otherOptions):
+               profileName=None, mozInfo=None, shuffle=False,
+               xunitFilename=None, xunitName=None, **otherOptions):
     """Run xpcshell tests.
 
     |xpcshell|, is the xpcshell executable to use to run the tests.
     |xrePath|, if provided, is the path to the XRE to use.
     |appPath|, if provided, is the path to an application directory.
     |symbolsPath|, if provided is the path to a directory containing
       breakpad symbols for processing crashes in tests.
     |manifest|, if provided, is a file containing a list of
@@ -412,20 +518,27 @@ class XPCShellTests(object):
     |logfiles|, if set to False, indicates not to save output to log files.
       Non-interactive only option.
     |debuggerInfo|, if set, specifies the debugger and debugger arguments
       that will be used to launch xpcshell.
     |profileName|, if set, specifies the name of the application for the profile
       directory if running only a subset of tests.
     |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict.
     |shuffle|, if True, execute tests in random order.
+    |xunitFilename|, if set, specifies the filename to which to write xUnit XML
+      results.
+    |xunitName|, if outputting an xUnit XML file, the str value to use for the
+      testsuite name.
     |otherOptions| may be present for the convenience of subclasses
     """
 
-    global gotSIGINT 
+    global gotSIGINT
+
+    if testdirs is None:
+        testdirs = []
 
     self.xpcshell = xpcshell
     self.xrePath = xrePath
     self.appPath = appPath
     self.symbolsPath = symbolsPath
     self.manifest = manifest
     self.testdirs = testdirs
     self.testPath = testPath
@@ -468,31 +581,39 @@ class XPCShellTests(object):
     
     pStdout, pStderr = self.getPipes()
 
     self.buildTestList()
 
     if shuffle:
       random.shuffle(self.alltests)
 
+    xunitResults = []
+
     for test in self.alltests:
       name = test['path']
       if self.singleFile and not name.endswith(self.singleFile):
         continue
 
       if self.testPath and name.find(self.testPath) == -1:
         continue
 
       self.testCount += 1
 
+      xunitResult = {"classname": "xpcshell", "name": test["name"]}
+
       # Check for skipped tests
       if 'disabled' in test:
         self.log.info("TEST-INFO | skipping %s | %s" %
                       (name, test['disabled']))
+
+        xunitResult["skipped"] = True
+        xunitResults.append(xunitResult)
         continue
+
       # Check for known-fail tests
       expected = test['expected'] == 'pass'
 
       testdir = os.path.dirname(name)
       self.buildXpcsCmd(testdir)
       testHeadFiles = self.getHeadFiles(test)
       testTailFiles = self.getTailFiles(test)
       cmdH = self.buildCmdHead(testHeadFiles, testTailFiles, self.xpcsCmd)
@@ -536,24 +657,39 @@ class XPCShellTests(object):
 
         result = not ((self.getReturnCode(proc) != 0) or
                       (stdout and re.search("^((parent|child): )?TEST-UNEXPECTED-",
                                             stdout, re.MULTILINE)) or
                       (stdout and re.search(": SyntaxError:", stdout,
                                             re.MULTILINE)))
 
         if result != expected:
-          self.log.error("TEST-UNEXPECTED-%s | %s | test failed (with xpcshell return code: %d), see following log:" % ("FAIL" if expected else "PASS", name, self.getReturnCode(proc)))
+          failureType = "TEST-UNEXPECTED-%s" % ("FAIL" if expected else "PASS")
+          message = "%s | %s | test failed (with xpcshell return code: %d), see following log:" % (
+                        failureType, name, self.getReturnCode(proc))
+          self.log.error(message)
           print_stdout(stdout)
           self.failCount += 1
+          xunitResult["passed"] = False
+
+          xunitResult["failure"] = {
+            "type": failureType,
+            "message": message,
+            "text": stdout
+          }
         else:
-          timeTaken = (time.time() - startTime) * 1000
+          now = time.time()
+          timeTaken = (now - startTime) * 1000
+          xunitResult["time"] = now - startTime
           self.log.info("TEST-%s | %s | test passed (time: %.3fms)" % ("PASS" if expected else "KNOWN-FAIL", name, timeTaken))
           if verbose:
             print_stdout(stdout)
+
+          xunitResult["passed"] = True
+
           if expected:
             self.passCount += 1
           else:
             self.todoCount += 1
 
         checkForCrashes(testdir, self.symbolsPath, testName=name)
         # Find child process(es) leak log(s), if any: See InitLog() in
         # xpcom/base/nsTraceRefcntImpl.cpp for logfile naming logic
@@ -567,34 +703,51 @@ class XPCShellTests(object):
         if self.logfiles and stdout:
           self.createLogFile(name, stdout, leakLogs)
       finally:
         # We don't want to delete the profile when running check-interactive
         # or check-one.
         if self.profileDir and not self.interactive and not self.singleFile:
           self.removeDir(self.profileDir)
       if gotSIGINT:
+        xunitResult["passed"] = False
+        xunitResult["time"] = "0.0"
+        xunitResult["failure"] = {
+          "type": "SIGINT",
+          "message": "Received SIGINT",
+          "text": "Received SIGINT (control-C) during test execution."
+        }
+
         self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution")
         if (keepGoing):
           gotSIGINT = False
         else:
+          xunitResults.append(xunitResult)
           break
+
+      xunitResults.append(xunitResult)
+
     if self.testCount == 0:
       self.log.error("TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?")
       self.failCount = 1
 
     self.log.info("""INFO | Result summary:
 INFO | Passed: %d
 INFO | Failed: %d
 INFO | Todo: %d""" % (self.passCount, self.failCount, self.todoCount))
 
+    if xunitFilename is not None:
+        self.writeXunitResults(filename=xunitFilename, results=xunitResults,
+                               name=xunitName)
+
     if gotSIGINT and not keepGoing:
       self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \
                      "(Use --keep-going to keep running tests after killing one with SIGINT)")
       return False
+
     return self.failCount == 0
 
 class XPCShellOptions(OptionParser):
   def __init__(self):
     """Process command line arguments and call runTests() to do the real work."""
     OptionParser.__init__(self)
 
     addCommonOptions(self)
@@ -632,16 +785,22 @@ class XPCShellOptions(OptionParser):
                     type = "string", dest="profileName", default=None,
                     help="name of application profile being tested")
     self.add_option("--build-info-json",
                     type = "string", dest="mozInfo", default=None,
                     help="path to a mozinfo.json including information about the build configuration. defaults to looking for mozinfo.json next to the script.")
     self.add_option("--shuffle",
                     action="store_true", dest="shuffle", default=False,
                     help="Execute tests in random order")
+    self.add_option("--xunit-file", dest="xunitFilename",
+                    help="path to file where xUnit results will be written.")
+    self.add_option("--xunit-suite-name", dest="xunitName",
+                    help="name to record for this xUnit test suite. Many "
+                         "tools expect Java class notation, e.g. "
+                         "dom.basic.foo")
 
 def main():
   parser = XPCShellOptions()
   options, args = parser.parse_args()
 
   if len(args) < 2 and options.manifest is None or \
      (len(args) < 1 and options.manifest is not None):
      print >>sys.stderr, """Usage: %s <path to xpcshell> <test dirs>
--- a/testing/xpcshell/selftest.py
+++ b/testing/xpcshell/selftest.py
@@ -2,16 +2,17 @@
 #
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 #
 
 from __future__ import with_statement
 import sys, os, unittest, tempfile, shutil
 from StringIO import StringIO
+from xml.etree.ElementTree import ElementTree
 
 from runxpcshelltests import XPCShellTests
 
 objdir = os.path.abspath(os.environ["OBJDIR"])
 xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
 if sys.platform == "win32":
     xpcshellBin += ".exe"
 
@@ -56,26 +57,27 @@ class XPCShellTestsTests(unittest.TestCa
                 testlines.extend(t[1:])
         self.manifest = self.writeFile("xpcshell.ini", """
 [DEFAULT]
 head =
 tail =
 
 """ + "\n".join(testlines))
 
-    def assertTestResult(self, expected, mozInfo={}, shuffle=False):
+    def assertTestResult(self, expected, shuffle=False, xunitFilename=None):
         """
         Assert that self.x.runTests with manifest=self.manifest
         returns |expected|.
         """
         self.assertEquals(expected,
                           self.x.runTests(xpcshellBin,
                                           manifest=self.manifest,
-                                          mozInfo=mozInfo,
-                                          shuffle=shuffle),
+                                          mozInfo={},
+                                          shuffle=shuffle,
+                                          xunitFilename=xunitFilename),
                           msg="""Tests should have %s, log:
 ========
 %s
 ========
 """ % ("passed" if expected else "failed", self.log.getvalue()))
 
     def _assertLog(self, s, expected):
         l = self.log.getvalue()
@@ -219,10 +221,54 @@ tail =
             self.writeFile(filename, SIMPLE_PASSING_TEST)
             manifest.append(filename)
 
         self.writeManifest(manifest)
         self.assertTestResult(True, shuffle=True)
         self.assertEquals(10, self.x.testCount)
         self.assertEquals(10, self.x.passCount)
 
+    def testXunitOutput(self):
+        """
+        Check that Xunit XML files are written.
+        """
+        self.writeFile("test_00.js", SIMPLE_PASSING_TEST)
+        self.writeFile("test_01.js", SIMPLE_FAILING_TEST)
+        self.writeFile("test_02.js", SIMPLE_PASSING_TEST)
+
+        manifest = [
+            "test_00.js",
+            "test_01.js",
+            ("test_02.js", "skip-if = true")
+        ]
+
+        self.writeManifest(manifest)
+
+        filename = os.path.join(self.tempdir, "xunit.xml")
+
+        self.assertTestResult(False, xunitFilename=filename)
+
+        self.assertTrue(os.path.exists(filename))
+        self.assertTrue(os.path.getsize(filename) > 0)
+
+        tree = ElementTree()
+        tree.parse(filename)
+        suite = tree.getroot()
+
+        self.assertTrue(suite is not None)
+        self.assertEqual(suite.get("tests"), "3")
+        self.assertEqual(suite.get("failures"), "1")
+        self.assertEqual(suite.get("skip"), "1")
+
+        testcases = suite.findall("testcase")
+        self.assertEqual(len(testcases), 3)
+
+        for testcase in testcases:
+            attributes = testcase.keys()
+            self.assertTrue("classname" in attributes)
+            self.assertTrue("name" in attributes)
+            self.assertTrue("time" in attributes)
+
+        self.assertTrue(testcases[1].find("failure") is not None)
+        self.assertTrue(testcases[2].find("skipped") is not None)
+
 if __name__ == "__main__":
     unittest.main()