Bug 729098 - Ability to track and rerun failed xpcshell tests; r=gps
authorChris Manchester <cmanchester@mozilla.com>
Wed, 21 Aug 2013 09:26:46 -0700
changeset 156940 0b6aa8fab175e2dbd65f23c337d5c18765a331cc
parent 156939 c82942addc204f43ab62b4547891629a7085af98
child 156941 a970626be5c9f1b918ff3bfd8ba7eb300b5bde91
push id2961
push userlsblakk@mozilla.com
push dateMon, 28 Oct 2013 21:59:28 +0000
treeherdermozilla-beta@73ef4f13486f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs729098
milestone26.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 729098 - Ability to track and rerun failed xpcshell tests; r=gps
testing/xpcshell/mach_commands.py
testing/xpcshell/runxpcshelltests.py
--- a/testing/xpcshell/mach_commands.py
+++ b/testing/xpcshell/mach_commands.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # Integrates the xpcshell test runner with mach.
 
 from __future__ import unicode_literals, print_function
 
 import mozpack.path
 import os
+import shutil
 import sys
 
 from StringIO import StringIO
 
 from mozbuild.base import (
     MachCommandBase,
     MozbuildObject,
 )
@@ -41,28 +42,30 @@ class XPCShellRunner(MozbuildObject):
     def run_suite(self, **kwargs):
         manifest = os.path.join(self.topobjdir, '_tests', 'xpcshell',
             'xpcshell.ini')
 
         return self._run_xpcshell_harness(manifest=manifest, **kwargs)
 
     def run_test(self, test_file, interactive=False,
                  keep_going=False, sequential=False, shuffle=False,
-                 debugger=None, debuggerArgs=None, debuggerInteractive=None):
+                 debugger=None, debuggerArgs=None, debuggerInteractive=None,
+                 rerun_failures=False):
         """Runs an individual xpcshell test."""
         # TODO Bug 794506 remove once mach integrates with virtualenv.
         build_path = os.path.join(self.topobjdir, 'build')
         if build_path not in sys.path:
             sys.path.append(build_path)
 
         if test_file == 'all':
             self.run_suite(interactive=interactive,
                            keep_going=keep_going, shuffle=shuffle, sequential=sequential,
                            debugger=debugger, debuggerArgs=debuggerArgs,
-                           debuggerInteractive=debuggerInteractive)
+                           debuggerInteractive=debuggerInteractive,
+                           rerun_failures=rerun_failures)
             return
 
         path_arg = self._wrap_path_argument(test_file)
 
         test_obj_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell',
             path_arg.relpath())
         if os.path.isfile(test_obj_dir):
             test_obj_dir = mozpack.path.dirname(test_obj_dir)
@@ -83,27 +86,29 @@ class XPCShellRunner(MozbuildObject):
             'interactive': interactive,
             'keep_going': keep_going,
             'shuffle': shuffle,
             'sequential': sequential,
             'test_dirs': xpcshell_dirs,
             'debugger': debugger,
             'debuggerArgs': debuggerArgs,
             'debuggerInteractive': debuggerInteractive,
+            'rerun_failures': rerun_failures
         }
 
         if os.path.isfile(path_arg.srcdir_path()):
             args['test_path'] = mozpack.path.basename(path_arg.srcdir_path())
 
         return self._run_xpcshell_harness(**args)
 
     def _run_xpcshell_harness(self, test_dirs=None, manifest=None,
                               test_path=None, shuffle=False, interactive=False,
                               keep_going=False, sequential=False,
-                              debugger=None, debuggerArgs=None, debuggerInteractive=None):
+                              debugger=None, debuggerArgs=None, debuggerInteractive=None,
+                              rerun_failures=False):
 
         # Obtain a reference to the xpcshell test runner.
         import runxpcshelltests
 
         dummy_log = StringIO()
         xpcshell = runxpcshelltests.XPCShellTests(log=dummy_log)
         self.log_manager.enable_unstructured()
 
@@ -139,16 +144,32 @@ class XPCShellRunner(MozbuildObject):
             else:
                 args['testdirs'] = [test_dirs]
         else:
             raise Exception('One of test_dirs or manifest must be provided.')
 
         if test_path is not None:
             args['testPath'] = test_path
 
+        # A failure manifest is written by default. If --rerun-failures is
+        # specified and a prior failure manifest is found, the prior manifest
+        # will be run. A new failure manifest is always written over any
+        # prior failure manifest.
+        failure_manifest_path = os.path.join(self.statedir, 'xpcshell.failures.ini')
+        rerun_manifest_path = os.path.join(self.statedir, 'xpcshell.rerun.ini')
+        if os.path.exists(failure_manifest_path) and rerun_failures:
+            shutil.move(failure_manifest_path, rerun_manifest_path)
+            args['manifest'] = rerun_manifest_path
+        elif os.path.exists(failure_manifest_path):
+            os.remove(failure_manifest_path)
+        elif rerun_failures:
+            print("No failures were found to re-run.")
+            return 0
+        args['failureManifest'] = failure_manifest_path
+
         # Python through 2.7.2 has issues with unicode in some of the
         # arguments. Work around that.
         filtered_args = {}
         for k, v in args.items():
             if isinstance(v, unicode_type):
                 v = v.encode('utf-8')
 
             if isinstance(k, unicode_type):
@@ -186,16 +207,18 @@ class MachCommands(MachCommandBase):
     @CommandArgument('--interactive', '-i', action='store_true',
         help='Open an xpcshell prompt before running tests.')
     @CommandArgument('--keep-going', '-k', action='store_true',
         help='Continue running tests after a SIGINT is received.')
     @CommandArgument('--sequential', action='store_true',
         help='Run the tests sequentially.')
     @CommandArgument('--shuffle', '-s', action='store_true',
         help='Randomize the execution order of tests.')
+    @CommandArgument('--rerun-failures', action='store_true',
+        help='Reruns failures from last time.')
     def run_xpcshell_test(self, **params):
         # We should probably have a utility function to ensure the tree is
         # ready to run tests. Until then, we just create the state dir (in
         # case the tree wasn't built with mach).
         self._ensure_state_subdir_exists('.')
 
         xpcshell = self._spawn(XPCShellRunner)
 
--- a/testing/xpcshell/runxpcshelltests.py
+++ b/testing/xpcshell/runxpcshelltests.py
@@ -106,16 +106,17 @@ class XPCShellTestThread(Thread):
         self.testharnessdir = kwargs.get('testharnessdir')
         self.profileName = kwargs.get('profileName')
         self.singleFile = kwargs.get('singleFile')
         self.env = copy.deepcopy(kwargs.get('env'))
         self.symbolsPath = kwargs.get('symbolsPath')
         self.logfiles = kwargs.get('logfiles')
         self.xpcshell = kwargs.get('xpcshell')
         self.xpcsRunArgs = kwargs.get('xpcsRunArgs')
+        self.failureManifest = kwargs.get('failureManifest')
 
         self.tests_root_dir = tests_root_dir
         self.app_dir_key = app_dir_key
         self.interactive = interactive
         self.verbose = verbose
         self.pStdout = pStdout
         self.pStderr = pStderr
         self.keep_going = keep_going
@@ -569,16 +570,23 @@ class XPCShellTestThread(Thread):
                 self.failCount += 1
                 self.xunit_result["passed"] = False
 
                 self.xunit_result["failure"] = {
                   "type": failureType,
                   "message": message,
                   "text": stdout
                 }
+
+                if self.failureManifest:
+                    with open(self.failureManifest, 'a') as f:
+                        f.write('[%s]\n' % self.test_object['path'])
+                        for k, v in self.test_object.items():
+                            f.write('%s = %s\n' % (k, v))
+
             else:
                 now = time.time()
                 timeTaken = (now - startTime) * 1000
                 self.xunit_result["time"] = now - startTime
 
                 with LOG_MUTEX:
                     self.log.info("TEST-%s | %s | test passed (time: %.3fms)" % ("PASS" if expected else "KNOWN-FAIL", name, timeTaken))
                     if self.verbose:
@@ -1057,17 +1065,18 @@ class XPCShellTests(object):
     def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None,
                  manifest=None, testdirs=None, testPath=None, mobileArgs=None,
                  interactive=False, verbose=False, keepGoing=False, logfiles=True,
                  thisChunk=1, totalChunks=1, debugger=None,
                  debuggerArgs=None, debuggerInteractive=False,
                  profileName=None, mozInfo=None, sequential=False, shuffle=False,
                  testsRootDir=None, xunitFilename=None, xunitName=None,
                  testingModulesDir=None, autolog=False, pluginsPath=None,
-                 testClass=XPCShellTestThread, **otherOptions):
+                 testClass=XPCShellTestThread, failureManifest=None,
+                 **otherOptions):
         """Run xpcshell tests.
 
         |xpcshell|, is the xpcshell executable to use to run the tests.
         |xrePath|, if provided, is the path to the XRE to use.
         |appPath|, if provided, is the path to an application directory.
         |symbolsPath|, if provided is the path to a directory containing
           breakpad symbols for processing crashes in tests.
         |manifest|, if provided, is a file containing a list of
@@ -1221,16 +1230,17 @@ class XPCShellTests(object):
             'testharnessdir': self.testharnessdir,
             'profileName': self.profileName,
             'singleFile': self.singleFile,
             'env': self.env, # making a copy of this in the testthreads
             'symbolsPath': self.symbolsPath,
             'logfiles': self.logfiles,
             'xpcshell': self.xpcshell,
             'xpcsRunArgs': self.xpcsRunArgs,
+            'failureManifest': failureManifest
         }
 
         if self.sequential:
             # Allow user to kill hung xpcshell subprocess with SIGINT
             # when we are only running tests sequentially.
             signal.signal(signal.SIGINT, markGotSIGINT)
 
         if self.debuggerInfo:
@@ -1435,16 +1445,19 @@ class XPCShellOptions(OptionParser):
                         action="store_true", dest="shuffle", default=False,
                         help="Execute tests in random order")
         self.add_option("--xunit-file", dest="xunitFilename",
                         help="path to file where xUnit results will be written.")
         self.add_option("--xunit-suite-name", dest="xunitName",
                         help="name to record for this xUnit test suite. Many "
                              "tools expect Java class notation, e.g. "
                              "dom.basic.foo")
+        self.add_option("--failure-manifest", dest="failureManifest",
+                        action="store",
+                        help="path to file where failure manifest will be written.")
 
 def main():
     parser = XPCShellOptions()
     options, args = parser.parse_args()
 
     if len(args) < 2 and options.manifest is None or \
        (len(args) < 1 and options.manifest is not None):
         print >>sys.stderr, """Usage: %s <path to xpcshell> <test dirs>