Add --verify mode for xpcshell tests; r=
authorGeoff Brown <gbrown@mozilla.com>
Wed, 02 Aug 2017 10:00:07 -0600
changeset 1204052 231e345e1a28cbb254535e04fc3fbcf54c30d657
parent 1203227 d41ed27736b2d8306e9c08d566e5057b6d11efa6
child 1204053 036725adc80a25225ff07bb52da14946e036445f
push id210013
push usergbrown@mozilla.com
push dateWed, 02 Aug 2017 16:00:45 +0000
treeherdertry@c2530a6a2222 [default view] [failures only]
milestone56.0a1
Add --verify mode for xpcshell tests; r=
testing/xpcshell/mach_commands.py
testing/xpcshell/runxpcshelltests.py
testing/xpcshell/xpcshellcommandline.py
--- a/testing/xpcshell/mach_commands.py
+++ b/testing/xpcshell/mach_commands.py
@@ -133,17 +133,17 @@ class XPCShellRunner(MozbuildObject):
                 k = k.encode('utf-8')
 
             filtered_args[k] = v
 
         result = xpcshell.runTests(**filtered_args)
 
         self.log_manager.disable_unstructured()
 
-        if not result and not xpcshell.sequential:
+        if not result and not xpcshell.sequential and not filtered_args['verify']:
             print("Tests were run in parallel. Try running with --sequential "
                   "to make sure the failures were not caused by this.")
         return int(not result)
 
 
 class AndroidXPCShellRunner(MozbuildObject):
     """Get specified DeviceManager"""
     def get_devicemanager(self, ip, port, remote_test_root):
--- a/testing/xpcshell/runxpcshelltests.py
+++ b/testing/xpcshell/runxpcshelltests.py
@@ -17,16 +17,17 @@ import shutil
 import signal
 import sys
 import tempfile
 import time
 import traceback
 
 from argparse import ArgumentParser
 from collections import defaultdict, deque, namedtuple
+from datetime import datetime, timedelta
 from distutils import dir_util
 from functools import partial
 from multiprocessing import cpu_count
 from subprocess import Popen, PIPE, STDOUT
 from tempfile import mkdtemp, gettempdir
 from threading import (
     Timer,
     Thread,
@@ -844,28 +845,28 @@ class XPCShellTests(object):
             test_object['manifest'] = os.path.relpath(test_object['manifest'], root)
 
         if os.sep != '/':
             for key in ('id', 'manifest'):
                 test_object[key] = test_object[key].replace(os.sep, '/')
 
         return test_object
 
-    def buildTestList(self, test_tags=None, test_paths=None):
+    def buildTestList(self, test_tags=None, test_paths=None, verify=False):
         """
           read the xpcshell.ini manifest and set self.alltests to be
           an array of test objects.
 
           if we are chunking tests, it will be done here as well
         """
 
         if test_paths is None:
             test_paths = []
 
-        if len(test_paths) == 1 and test_paths[0].endswith(".js"):
+        if len(test_paths) == 1 and test_paths[0].endswith(".js") and not verify:
             self.singleFile = os.path.basename(test_paths[0])
         else:
             self.singleFile = None
 
         mp = self.getTestManifest(self.manifest)
 
         root = mp.rootdir
         if build and not root:
@@ -1120,17 +1121,17 @@ class XPCShellTests(object):
                  thisChunk=1, totalChunks=1, debugger=None,
                  debuggerArgs=None, debuggerInteractive=False,
                  profileName=None, mozInfo=None, sequential=False, shuffle=False,
                  testingModulesDir=None, pluginsPath=None,
                  testClass=XPCShellTestThread, failureManifest=None,
                  log=None, stream=None, jsDebugger=False, jsDebuggerPort=0,
                  test_tags=None, dump_tests=None, utility_path=None,
                  rerun_failures=False, threadCount=NUM_THREADS,
-                 failure_manifest=None, jscovdir=None, **otherOptions):
+                 failure_manifest=None, jscovdir=None, verify=False, verifyMaxTime=3600, **otherOptions):
         """Run xpcshell tests.
 
         |xpcshell|, is the xpcshell executable to use to run the tests.
         |xrePath|, if provided, is the path to the XRE to use.
         |appPath|, if provided, is the path to an application directory.
         |symbolsPath|, if provided is the path to a directory containing
           breakpad symbols for processing crashes in tests.
         |manifest|, if provided, is a file containing a list of
@@ -1152,21 +1153,25 @@ class XPCShellTests(object):
           mode.
         |profileName|, if set, specifies the name of the application for the profile
           directory if running only a subset of tests.
         |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict.
         |shuffle|, if True, execute tests in random order.
         |testingModulesDir|, if provided, specifies where JS modules reside.
           xpcshell will register a resource handler mapping this path.
         |tempDir|, if provided, specifies a temporary directory to use.
+        |verify|, if True, run tests many times, to find intermittent failures.
         |otherOptions| may be present for the convenience of subclasses
         """
 
         global gotSIGINT
 
+        # Number of times to repeat test(s) when running with --verify
+        VERIFY_REPEAT = 10
+
         # Try to guess modules directory.
         # This somewhat grotesque hack allows the buildbot machines to find the
         # modules directory without having to configure the buildbot hosts. This
         # code path should never be executed in local runs because the build system
         # should always set this argument.
         if not testingModulesDir:
             possible = os.path.join(here, os.path.pardir, 'modules')
 
@@ -1279,25 +1284,24 @@ class XPCShellTests(object):
             appDirKey = self.mozInfo["appname"] + "-appdir"
 
         # We have to do this before we run tests that depend on having the node
         # http/2 server.
         self.trySetupNode()
 
         pStdout, pStderr = self.getPipes()
 
-        self.buildTestList(test_tags, testPaths)
+        self.buildTestList(test_tags, testPaths, verify)
         if self.singleFile:
             self.sequential = True
 
         if shuffle:
             random.shuffle(self.alltests)
 
         self.cleanup_dir_list = []
-        self.try_again_list = []
 
         kwargs = {
             'appPath': self.appPath,
             'xrePath': self.xrePath,
             'utility_path': self.utility_path,
             'testingModulesDir': self.testingModulesDir,
             'debuggerInfo': self.debuggerInfo,
             'jsDebuggerInfo': self.jsDebuggerInfo,
@@ -1348,51 +1352,150 @@ class XPCShellTests(object):
         # The test itself needs to know whether it is a tsan build, since
         # that has an effect on interpretation of the process return value.
         usingTSan = "tsan" in self.mozInfo and self.mozInfo["tsan"]
 
         # create a queue of all tests that will run
         tests_queue = deque()
         # also a list for the tests that need to be run sequentially
         sequential_tests = []
-        for test_object in self.alltests:
-            # Test identifiers are provided for the convenience of logging. These
-            # start as path names but are rewritten in case tests from the same path
-            # are re-run.
+        status = None
+        if not verify:
+            for test_object in self.alltests:
+                # Test identifiers are provided for the convenience of logging. These
+                # start as path names but are rewritten in case tests from the same path
+                # are re-run.
+
+                path = test_object['path']
+
+                if self.singleFile and not path.endswith(self.singleFile):
+                    continue
+
+                self.testCount += 1
+
+                test = testClass(test_object, self.event, self.cleanup_dir_list,
+                        app_dir_key=appDirKey,
+                        interactive=interactive,
+                        verbose=verbose or test_object.get("verbose") == "true",
+                        pStdout=pStdout, pStderr=pStderr,
+                        keep_going=keepGoing, log=self.log, usingTSan=usingTSan,
+                        mobileArgs=mobileArgs, **kwargs)
+                if 'run-sequentially' in test_object or self.sequential:
+                    sequential_tests.append(test)
+                else:
+                    tests_queue.append(test)
+
+            if self.sequential:
+                self.log.info("Running tests sequentially.")
+            else:
+                self.log.info("Using at most %d threads." % self.threadCount)
 
-            path = test_object['path']
+            status = self.runTestObjects(tests_queue, sequential_tests,
+                testClass, appDirKey, interactive,
+                verbose, pStdout, pStderr, keepGoing,
+                log, usingTSan, **kwargs)
+
+        else:
+            #
+            # Test verification: Run each test many times, in various configurations,
+            # in hopes of finding intermittent failures.
+            #
 
-            if self.singleFile and not path.endswith(self.singleFile):
-                continue
-
-            self.testCount += 1
+            def step1():
+                # Run tests sequentially. Parallel mode would also work, except that
+                # the logging system gets confused when 2 or more tests with the same
+                # name run at the same time.
+                sequential_tests = []
+                for i in xrange(VERIFY_REPEAT):
+                    self.testCount += 1
+                    test = testClass(test_object, self.event, self.cleanup_dir_list,
+                            retry=False,
+                            app_dir_key=appDirKey,
+                            interactive=interactive,
+                            verbose=test_object.get("verbose") == "true",
+                            pStdout=pStdout, pStderr=pStderr,
+                            keep_going=keepGoing, log=self.log, usingTSan=usingTSan,
+                            mobileArgs=mobileArgs, **kwargs)
+                    sequential_tests.append(test)
+                status = self.runTestObjects(tests_queue, sequential_tests,
+                    testClass, appDirKey, interactive,
+                    verbose, pStdout, pStderr, keepGoing,
+                    log, usingTSan, **kwargs)
+                return status
 
-            test = testClass(test_object, self.event, self.cleanup_dir_list,
-                    app_dir_key=appDirKey,
-                    interactive=interactive,
-                    verbose=verbose or test_object.get("verbose") == "true",
-                    pStdout=pStdout, pStderr=pStderr,
-                    keep_going=keepGoing, log=self.log, usingTSan=usingTSan,
-                    mobileArgs=mobileArgs, **kwargs)
-            if 'run-sequentially' in test_object or self.sequential:
-                sequential_tests.append(test)
-            else:
-                tests_queue.append(test)
+            def step2():
+                # Run tests sequentially, with MOZ_CHAOSMODE enabled.
+                sequential_tests = []
+                self.env["MOZ_CHAOSMODE"] = ""
+                for i in xrange(VERIFY_REPEAT):
+                    self.testCount += 1
+                    test = testClass(test_object, self.event, self.cleanup_dir_list,
+                            retry=False,
+                            app_dir_key=appDirKey,
+                            interactive=interactive,
+                            verbose=test_object.get("verbose") == "true",
+                            pStdout=pStdout, pStderr=pStderr,
+                            keep_going=keepGoing, log=self.log, usingTSan=usingTSan,
+                            mobileArgs=mobileArgs, **kwargs)
+                    sequential_tests.append(test)
+                status = self.runTestObjects(tests_queue, sequential_tests,
+                    testClass, appDirKey, interactive,
+                    verbose, pStdout, pStderr, keepGoing,
+                    log, usingTSan, **kwargs)
+                return status
 
-        if self.sequential:
-            self.log.info("Running tests sequentially.")
-        else:
-            self.log.info("Using at most %d threads." % self.threadCount)
+            steps = [
+                ("1. Run each test %d times, sequentially." % VERIFY_REPEAT,
+                 step1),
+                ("2. Run each test %d times, sequentially, in chaos mode." % VERIFY_REPEAT,
+                 step2),
+            ]
+            startTime = datetime.now()
+            maxTime = timedelta(seconds=verifyMaxTime)
+            for test_object in self.alltests:
+                stepResults = {}
+                for (descr, step) in steps:
+                    stepResults[descr] = "not run / incomplete"
+                finalResult = "PASSED"
+                for (descr, step) in steps:
+                    if (datetime.now() - startTime) > maxTime:
+                        self.log.info("::: Test verification is taking too long: Giving up!")
+                        self.log.info("::: So far, all checks passed, but not all checks were run.")
+                        break
+                    self.log.info(':::')
+                    self.log.info('::: Running test verification step "%s"...' % descr)
+                    self.log.info(':::')
+                    status = step()
+                    if status != True:
+                        stepResults[descr] = "FAIL"
+                        finalResult = "FAILED!"
+                        break
+                    stepResults[descr] = "Pass"
+                self.log.info(':::')
+                self.log.info('::: Test verification summary for: %s' % test_object['path'])
+                self.log.info(':::')
+                for descr in sorted(stepResults.keys()):
+                    self.log.info('::: %s : %s' % (descr, stepResults[descr]))
+                self.log.info(':::')
+                self.log.info('::: Test verification %s' % finalResult)
+                self.log.info(':::')
 
+        return status
+
+    def runTestObjects(self, tests_queue, sequential_tests,
+            testClass, appDirKey=None, interactive=False,
+            verbose=False, pStdout=None, pStderr=None, keepGoing=False,
+            log=None, usingTSan=False, mobileArgs=None, **kwargs):
         # keep a set of threadCount running tests and start running the
         # tests in the queue at most threadCount at a time
         running_tests = set()
         keep_going = True
         exceptions = []
         tracebacks = []
+        self.try_again_list = []
 
         tests_by_manifest = defaultdict(list)
         for test in self.alltests:
             tests_by_manifest[test['manifest']].append(test['id'])
         self.log.suite_start(tests_by_manifest)
 
         while tests_queue or running_tests:
             # if we're not supposed to continue and all of the running tests
--- a/testing/xpcshell/xpcshellcommandline.py
+++ b/testing/xpcshell/xpcshellcommandline.py
@@ -114,16 +114,23 @@ def add_common_arguments(parser):
                         "(with --rerun-failure) or in which to record failed tests")
     parser.add_argument("--threads",
                         type=int, dest="threadCount", default=0,
                         help="override the number of jobs (threads) when running tests in parallel, "
                              "the default is CPU x 1.5 when running via mach and CPU x 4 when running "
                              "in automation")
     parser.add_argument("testPaths", nargs="*", default=None,
                         help="Paths of tests to run.")
+    parser.add_argument("--verify",
+                        action="store_true", default=False,
+                        help="Verify test(s) by running multiple times.")
+    parser.add_argument("--verify-max-time",
+                        dest="verifyMaxTime",
+                        type=int, default=3600,
+                        help="Maximum time, in seconds, to run in --verify mode.")
 
 def add_remote_arguments(parser):
     parser.add_argument("--deviceIP", action="store", type=str, dest="deviceIP",
                         help="ip address of remote device to test")
 
     parser.add_argument("--devicePort", action="store", type=str, dest="devicePort",
                         default=20701, help="port of remote device to test")