Bug 1358978 - Implement --run-until-failure for Marionette. draft
authorHenrik Skupin <mail@hskupin.info>
Mon, 24 Apr 2017 12:48:33 +0200
changeset 567075 cf4206e57253d49d88db1325b0c9fbe240e61116
parent 567074 6efe7915f97b389c51b85bb2e1e45004f33ca680
child 625511 e1acc5ab2f92db9bc00ad7d730d90174882b0493
push id55429
push userbmo:hskupin@gmail.com
push dateMon, 24 Apr 2017 10:48:57 +0000
bugs1358978
milestone55.0a1
Bug 1358978 - Implement --run-until-failure for Marionette. To help debugging intermittent failures the --repeat option can be used, but that doesn't stop if a failure occurres and continuous until the number of specified iterations have been reached. With --run-until-failure we can allow the harness to stop running the tests once the first failure appeared. Without the --repeat option specified it will repeat 30 times by default. MozReview-Commit-ID: Jlsss4PHNbj
testing/marionette/harness/marionette_harness/runner/base.py
--- a/testing/marionette/harness/marionette_harness/runner/base.py
+++ b/testing/marionette/harness/marionette_harness/runner/base.py
@@ -299,16 +299,21 @@ class BaseMarionetteArguments(ArgumentPa
         self.add_argument('--addon',
                           action='append',
                           dest='addons',
                           help="addon to install; repeat for multiple addons.")
         self.add_argument('--repeat',
                           type=int,
                           default=0,
                           help='number of times to repeat the test(s)')
+        self.add_argument("--run-until-failure",
+                          action="store_true",
+                          help="Run tests repeatedly and stops on the first time a test fails. "
+                               "Default cap is 30 runs, which can be overwritten "
+                               "with the --repeat parameter.")
         self.add_argument('--testvars',
                           action='append',
                           help='path to a json file with any test data required')
         self.add_argument('--symbols-path',
                           help='absolute path to directory containing breakpad symbols, or the '
                                'url of a zip file containing symbols')
         self.add_argument('--startup-timeout',
                           type=int,
@@ -437,16 +442,19 @@ class BaseMarionetteArguments(ArgumentPa
             self.error('You must specify how many chunks to split the tests into.')
 
         if args.total_chunks is not None:
             if not 1 < args.total_chunks:
                 self.error('Total chunks must be greater than 1.')
             if not 1 <= args.this_chunk <= args.total_chunks:
                 self.error('Chunk to run must be between 1 and {}.'.format(args.total_chunks))
 
+        if args.run_until_failure and not args.repeat:
+            args.repeat = 30
+
         if args.jsdebugger:
             args.app_args.append('-jsdebugger')
             args.socket_timeout = None
 
         args.prefs = self._get_preferences(args.prefs_files, args.prefs_args)
 
         for container in self.argument_containers:
             if hasattr(container, 'verify_usage_handler'):
@@ -494,17 +502,19 @@ class Fixtures(object):
 class BaseMarionetteTestRunner(object):
 
     textrunnerclass = MarionetteTextTestRunner
     driverclass = Marionette
 
     def __init__(self, address=None,
                  app=None, app_args=None, binary=None, profile=None,
                  logger=None, logdir=None,
-                 repeat=0, testvars=None,
+                 repeat=0,
+                 run_until_failure=False,
+                 testvars=None,
                  symbols_path=None,
                  shuffle=False, shuffle_seed=random.randint(0, sys.maxint), this_chunk=1,
                  total_chunks=1,
                  server_root=None, gecko_log=None, result_callbacks=None,
                  prefs=None, test_tags=None,
                  socket_timeout=BaseMarionetteArguments.socket_timeout_default,
                  startup_timeout=None, addons=None, workspace=None,
                  verbose=0, e10s=True, emulator=False, **kwargs):
@@ -524,16 +534,17 @@ class BaseMarionetteTestRunner(object):
         self.bin = binary
         self.emulator = emulator
         self.profile = profile
         self.addons = addons
         self.logger = logger
         self.marionette = None
         self.logdir = logdir
         self.repeat = repeat
+        self.run_until_failure = run_until_failure
         self.symbols_path = symbols_path
         self.socket_timeout = socket_timeout
         self.shuffle = shuffle
         self.shuffle_seed = shuffle_seed
         self.server_root = server_root
         self.this_chunk = this_chunk
         self.total_chunks = total_chunks
         self.mixin_run_tests = []
@@ -852,23 +863,27 @@ class BaseMarionetteTestRunner(object):
         self.logger.suite_start(tests_by_group,
                                 version_info=self.version_info,
                                 device_info=device_info)
 
         self._log_skipped_tests()
 
         interrupted = None
         try:
-            counter = self.repeat
-            while counter >= 0:
-                round_num = self.repeat - counter
-                if round_num > 0:
-                    self.logger.info('\nREPEAT {}\n-------'.format(round_num))
+            repeat_index = 0
+            while repeat_index <= self.repeat:
+                if repeat_index > 0:
+                    self.logger.info("\nREPEAT {}\n-------".format(repeat_index))
                 self.run_test_sets()
-                counter -= 1
+
+                if self.results[repeat_index].failures or self.results[repeat_index].errors:
+                    break
+
+                repeat_index += 1
+
         except KeyboardInterrupt:
             # in case of KeyboardInterrupt during the test execution
             # we want to display current test results.
             # so we keep the exception to raise it later.
             interrupted = sys.exc_info()
         except:
             # For any other exception we return immediately and have to
             # cleanup running processes