Bug 984208 - [Marionette Client] Modifying shuffle to put its seed outside, r=mdas
authorLuv Agarwal <luvagarwal1995su@gmail.com>
Thu, 05 Jun 2014 11:14:49 -0400
changeset 206069 9347f1e2551ad78fdd8d84b569c2bacacfb1c4a5
parent 206068 6a527532be744d743fadc8cdf743351ba0d82da1
child 206070 ba505f7104351ab33b7253e05318caca3687d38e
push id3741
push userasasaki@mozilla.com
push dateMon, 21 Jul 2014 20:25:18 +0000
treeherdermozilla-beta@4d6f46f5af68 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmdas
bugs984208
milestone32.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 984208 - [Marionette Client] Modifying shuffle to put its seed outside, r=mdas
testing/marionette/client/marionette/runner/base.py
--- a/testing/marionette/client/marionette/runner/base.py
+++ b/testing/marionette/client/marionette/runner/base.py
@@ -466,16 +466,21 @@ class BaseMarionetteOptions(OptionParser
                         dest='es_servers',
                         action='append',
                         help='the ElasticSearch server to use for autolog submission')
         self.add_option('--shuffle',
                         action='store_true',
                         dest='shuffle',
                         default=False,
                         help='run tests in a random order')
+        self.add_option('--shuffle-seed',
+                        dest='shuffle_seed',
+                        type=int,
+                        default=random.randint(0, sys.maxint),
+                        help='Use given seed to shuffle tests')
         self.add_option('--total-chunks',
                         dest='total_chunks',
                         type=int,
                         help='how many chunks to split the tests up into')
         self.add_option('--this-chunk',
                         dest='this_chunk',
                         type=int,
                         help='which chunk to run')
@@ -559,17 +564,18 @@ class BaseMarionetteTestRunner(object):
 
     def __init__(self, address=None, emulator=None, emulatorBinary=None,
                  emulatorImg=None, emulator_res='480x800', homedir=None,
                  app=None, app_args=None, bin=None, profile=None, autolog=False,
                  revision=None, logger=None, testgroup="marionette", noWindow=False,
                  logcat_dir=None, xml_output=None, repeat=0, gecko_path=None,
                  testvars=None, tree=None, type=None, device_serial=None,
                  symbols_path=None, timeout=None, es_servers=None, shuffle=False,
-                 sdcard=None, this_chunk=1, total_chunks=1, sources=None, server_root=None,
+                 shuffle_seed=random.randint(0, sys.maxint), sdcard=None,
+                 this_chunk=1, total_chunks=1, sources=None, server_root=None,
                  gecko_log=None,
                  **kwargs):
         self.address = address
         self.emulator = emulator
         self.emulatorBinary = emulatorBinary
         self.emulatorImg = emulatorImg
         self.emulator_res = emulator_res
         self.homedir = homedir
@@ -595,16 +601,17 @@ class BaseMarionetteTestRunner(object):
         self.device_serial = device_serial
         self.symbols_path = symbols_path
         self.timeout = timeout
         self._device = None
         self._capabilities = None
         self._appName = None
         self.es_servers = es_servers
         self.shuffle = shuffle
+        self.shuffle_seed = shuffle_seed
         self.sdcard = sdcard
         self.sources = sources
         self.server_root = server_root
         self.this_chunk = this_chunk
         self.total_chunks = total_chunks
         self.gecko_log = gecko_log
         self.mixin_run_tests = []
         self.manifest_skipped_tests = []
@@ -855,16 +862,18 @@ class BaseMarionetteTestRunner(object):
         if self.marionette.instance:
             self.marionette.instance.close()
             self.marionette.instance = None
 
         self.marionette.cleanup()
 
         for run_tests in self.mixin_run_tests:
             run_tests(tests)
+        if self.shuffle:
+            self.logger.info("Using seed where seed is:%d" % self.shuffle_seed)
 
     def add_test(self, test, expected='pass', oop=None):
         filepath = os.path.abspath(test)
 
         if os.path.isdir(filepath):
             for root, dirs, files in os.walk(filepath):
                 for filename in files:
                     if ((filename.startswith('test_') or filename.startswith('browser_')) and
@@ -1006,16 +1015,17 @@ class BaseMarionetteTestRunner(object):
                 self.unexpected_successes += len(results.unexpectedSuccesses)
                 for failure in results.unexpectedSuccesses:
                     self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS'))
             if hasattr(results, 'expectedFailures'):
                 self.todo += len(results.expectedFailures)
 
     def run_test_set(self, tests):
         if self.shuffle:
+            random.seed(self.shuffle_seed)
             random.shuffle(tests)
 
         for test in tests:
             self.run_test(test['filepath'], test['expected'], test['oop'])
             if self.marionette.check_for_crash():
                 break
 
     def run_test_sets(self):