Bug 1390889 - Support --verify option in reftest harness; r=jmaher
authorGeoff Brown <gbrown@mozilla.com>
Tue, 05 Sep 2017 16:55:34 -0600
changeset 379126 34deee9270677e70e6dc2009874b468fe6ff28b6
parent 379125 870c7725fcccac8137acc6d254b2e4e0f67557f1
child 379127 7b30671832a4ef163accb749f3d90f272e2004b0
push id32449
push userarchaeopteryx@coole-files.de
push dateWed, 06 Sep 2017 09:33:20 +0000
treeherdermozilla-central@c959327c6b75 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1390889
milestone57.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1390889 - Support --verify option in reftest harness; r=jmaher
layout/tools/reftest/reftestcommandline.py
layout/tools/reftest/runreftest.py
--- a/layout/tools/reftest/reftestcommandline.py
+++ b/layout/tools/reftest/reftestcommandline.py
@@ -246,16 +246,26 @@ class ReftestArgumentsParser(argparse.Ar
                           dest="workPath",
                           help="Path to the base dir of all source files.")
 
         self.add_argument("--obj-path",
                           action="store",
                           dest="objPath",
                           help="Path to the base dir of all object files.")
 
+        self.add_argument("--verify",
+                          action="store_true",
+                          default=False,
+                          help="Test verification mode.")
+
+        self.add_argument("--verify-max-time",
+                          type=int,
+                          default=3600,
+                          help="Maximum time, in seconds, to run in --verify mode..")
+
         mozlog.commandline.add_logging_group(self)
 
     def get_ip(self):
         import moznetwork
         if os.name != "nt":
             return moznetwork.get_ip()
         else:
             self.error(
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -2,27 +2,29 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """
 Runs the reftest test harness.
 """
 
 import collections
+import copy
 import itertools
 import json
 import multiprocessing
 import os
 import platform
 import re
 import shutil
 import signal
 import subprocess
 import sys
 import threading
+from datetime import datetime, timedelta
 
 SCRIPT_DIRECTORY = os.path.abspath(
     os.path.realpath(os.path.dirname(__file__)))
 if SCRIPT_DIRECTORY not in sys.path:
     sys.path.insert(0, SCRIPT_DIRECTORY)
 
 import mozcrash
 import mozdebug
@@ -410,16 +412,89 @@ class RefTest(object):
         self.leakLogFile = os.path.join(profileDir, "runreftest_leaks.log")
         browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leakLogFile
         return browserEnv
 
     def cleanup(self, profileDir):
         if profileDir:
             shutil.rmtree(profileDir, True)
 
+    def verifyTests(self, tests, options):
+        """
+        Support --verify mode: Run test(s) many times in a variety of
+        configurations/environments in an effort to find intermittent
+        failures.
+        """
+
+        self._populate_logger(options)
+
+        # Number of times to repeat test(s) when running with --repeat
+        VERIFY_REPEAT = 20
+        # Number of times to repeat test(s) when running test in separate browser
+        VERIFY_REPEAT_SINGLE_BROWSER = 10
+
+        def step1():
+            stepOptions = copy.deepcopy(options)
+            stepOptions.repeat = VERIFY_REPEAT
+            stepOptions.runUntilFailure = True
+            result = self.runTests(tests, stepOptions)
+            return result
+
+        def step2():
+            stepOptions = copy.deepcopy(options)
+            for i in xrange(VERIFY_REPEAT_SINGLE_BROWSER):
+                result = self.runTests(tests, stepOptions)
+                if result != 0:
+                    break
+            return result
+
+        steps = [
+            ("1. Run each test %d times in one browser." % VERIFY_REPEAT,
+             step1),
+            ("2. Run each test %d times in a new browser each time." %
+             VERIFY_REPEAT_SINGLE_BROWSER,
+             step2),
+        ]
+
+        stepResults = {}
+        for (descr, step) in steps:
+            stepResults[descr] = "not run / incomplete"
+
+        startTime = datetime.now()
+        maxTime = timedelta(seconds=options.verify_max_time)
+        finalResult = "PASSED"
+        for (descr, step) in steps:
+            if (datetime.now() - startTime) > maxTime:
+                self.log.info("::: Test verification is taking too long: Giving up!")
+                self.log.info("::: So far, all checks passed, but not all checks were run.")
+                break
+            self.log.info(':::')
+            self.log.info('::: Running test verification step "%s"...' % descr)
+            self.log.info(':::')
+            result = step()
+            if result != 0:
+                stepResults[descr] = "FAIL"
+                finalResult = "FAILED!"
+                break
+            stepResults[descr] = "Pass"
+
+        self.log.info(':::')
+        self.log.info('::: Test verification summary for:')
+        self.log.info(':::')
+        for test in tests:
+            self.log.info('::: '+test)
+        self.log.info(':::')
+        for descr in sorted(stepResults.keys()):
+            self.log.info('::: %s : %s' % (descr, stepResults[descr]))
+        self.log.info(':::')
+        self.log.info('::: Test verification %s' % finalResult)
+        self.log.info(':::')
+
+        return result
+
     def runTests(self, tests, options, cmdargs=None):
         cmdargs = cmdargs or []
         self._populate_logger(options)
 
         if options.cleanupCrashes:
             mozcrash.cleanup_pending_crash_reports()
 
         manifests = self.resolver.resolveManifests(options, tests)
@@ -758,15 +833,20 @@ def run_test_harness(parser, options):
     options.app = reftest.getFullPath(options.app)
     if not os.path.exists(options.app):
         parser.error("Error: Path %(app)s doesn't exist. Are you executing "
                      "$objdir/_tests/reftest/runreftest.py?" % {"app": options.app})
 
     if options.xrePath is None:
         options.xrePath = os.path.dirname(options.app)
 
-    return reftest.runTests(options.tests, options)
+    if options.verify:
+        result = reftest.verifyTests(options.tests, options)
+    else:
+        result = reftest.runTests(options.tests, options)
+
+    return result
 
 
 if __name__ == "__main__":
     parser = reftestcommandline.DesktopArgumentsParser()
     options = parser.parse_args()
     sys.exit(run_test_harness(parser, options))