--- a/testing/marionette/harness/marionette_harness/runner/base.py
+++ b/testing/marionette/harness/marionette_harness/runner/base.py
@@ -299,16 +299,21 @@ class BaseMarionetteArguments(ArgumentPa
self.add_argument('--addon',
action='append',
dest='addons',
help="addon to install; repeat for multiple addons.")
self.add_argument('--repeat',
type=int,
default=0,
help='number of times to repeat the test(s)')
+ self.add_argument("--run-until-failure",
+ action="store_true",
+ help="Run tests repeatedly and stops on the first time a test fails. "
+ "Default cap is 30 runs, which can be overwritten "
+ "with the --repeat parameter.")
self.add_argument('--testvars',
action='append',
help='path to a json file with any test data required')
self.add_argument('--symbols-path',
help='absolute path to directory containing breakpad symbols, or the '
'url of a zip file containing symbols')
self.add_argument('--startup-timeout',
type=int,
@@ -437,16 +442,19 @@ class BaseMarionetteArguments(ArgumentPa
self.error('You must specify how many chunks to split the tests into.')
if args.total_chunks is not None:
if not 1 < args.total_chunks:
self.error('Total chunks must be greater than 1.')
if not 1 <= args.this_chunk <= args.total_chunks:
self.error('Chunk to run must be between 1 and {}.'.format(args.total_chunks))
+ if args.run_until_failure and not args.repeat:
+ args.repeat = 30
+
if args.jsdebugger:
args.app_args.append('-jsdebugger')
args.socket_timeout = None
args.prefs = self._get_preferences(args.prefs_files, args.prefs_args)
for container in self.argument_containers:
if hasattr(container, 'verify_usage_handler'):
@@ -494,17 +502,19 @@ class Fixtures(object):
class BaseMarionetteTestRunner(object):
textrunnerclass = MarionetteTextTestRunner
driverclass = Marionette
def __init__(self, address=None,
app=None, app_args=None, binary=None, profile=None,
logger=None, logdir=None,
- repeat=0, testvars=None,
+ repeat=0,
+ run_until_failure=False,
+ testvars=None,
symbols_path=None,
shuffle=False, shuffle_seed=random.randint(0, sys.maxint), this_chunk=1,
total_chunks=1,
server_root=None, gecko_log=None, result_callbacks=None,
prefs=None, test_tags=None,
socket_timeout=BaseMarionetteArguments.socket_timeout_default,
startup_timeout=None, addons=None, workspace=None,
verbose=0, e10s=True, emulator=False, **kwargs):
@@ -524,16 +534,17 @@ class BaseMarionetteTestRunner(object):
self.bin = binary
self.emulator = emulator
self.profile = profile
self.addons = addons
self.logger = logger
self.marionette = None
self.logdir = logdir
self.repeat = repeat
+ self.run_until_failure = run_until_failure
self.symbols_path = symbols_path
self.socket_timeout = socket_timeout
self.shuffle = shuffle
self.shuffle_seed = shuffle_seed
self.server_root = server_root
self.this_chunk = this_chunk
self.total_chunks = total_chunks
self.mixin_run_tests = []
@@ -852,23 +863,27 @@ class BaseMarionetteTestRunner(object):
self.logger.suite_start(tests_by_group,
version_info=self.version_info,
device_info=device_info)
self._log_skipped_tests()
interrupted = None
try:
- counter = self.repeat
- while counter >= 0:
- round_num = self.repeat - counter
- if round_num > 0:
- self.logger.info('\nREPEAT {}\n-------'.format(round_num))
+ repeat_index = 0
+ while repeat_index <= self.repeat:
+ if repeat_index > 0:
+ self.logger.info("\nREPEAT {}\n-------".format(repeat_index))
self.run_test_sets()
- counter -= 1
+
+ if self.results[repeat_index].failures or self.results[repeat_index].errors:
+ break
+
+ repeat_index += 1
+
except KeyboardInterrupt:
# in case of KeyboardInterrupt during the test execution
# we want to display current test results.
# so we keep the exception to raise it later.
interrupted = sys.exc_info()
except:
# For any other exception we return immediately and have to
# cleanup running processes