Bug 912890 - [Marionette Client] Add shuffle for tests. r=jgriffin
authorAskeing Yen <fyen@mozilla.com>
Fri, 06 Sep 2013 15:30:55 +0800
changeset 146149 2dd351292f7cc21f6e657bb285b5b98b3984d28b
parent 146148 035ce07737626ce7fa3b5651d0b63b7e30798e05
child 146150 b1f2748e657cb7e72de1c6b91f930b5ea1faac24
push id25243
push userryanvm@gmail.com
push dateMon, 09 Sep 2013 19:52:36 +0000
treeherdermozilla-central@9edc229b7d09 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgriffin
bugs912890
milestone26.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 912890 - [Marionette Client] Add shuffle for tests. r=jgriffin
testing/marionette/client/marionette/runtests.py
--- a/testing/marionette/client/marionette/runtests.py
+++ b/testing/marionette/client/marionette/runtests.py
@@ -6,16 +6,17 @@ from datetime import datetime
 import logging
 from optparse import OptionParser
 import os
 import unittest
 import socket
 import sys
 import time
 import traceback
+import random
 import moznetwork
 import xml.dom.minidom as dom
 
 from manifestparser import TestManifest
 from mozhttpd import MozHttpd
 
 from marionette import Marionette
 from marionette_test import MarionetteJSTestCase, MarionetteTestCase
@@ -205,17 +206,17 @@ class MarionetteTestRunner(object):
     textrunnerclass = MarionetteTextTestRunner
 
     def __init__(self, address=None, emulator=None, emulatorBinary=None,
                  emulatorImg=None, emulator_res='480x800', homedir=None,
                  app=None, app_args=None, bin=None, profile=None, autolog=False,
                  revision=None, logger=None, testgroup="marionette", noWindow=False,
                  logcat_dir=None, xml_output=None, repeat=0, gecko_path=None,
                  testvars=None, tree=None, type=None, device_serial=None,
-                 symbols_path=None, timeout=None, es_servers=None, **kwargs):
+                 symbols_path=None, timeout=None, es_servers=None, shuffle=False, **kwargs):
         self.address = address
         self.emulator = emulator
         self.emulatorBinary = emulatorBinary
         self.emulatorImg = emulatorImg
         self.emulator_res = emulator_res
         self.homedir = homedir
         self.app = app
         self.app_args = app_args or []
@@ -239,16 +240,17 @@ class MarionetteTestRunner(object):
         self.type = type
         self.device_serial = device_serial
         self.symbols_path = symbols_path
         self.timeout = timeout
         self._device = None
         self._capabilities = None
         self._appName = None
         self.es_servers = es_servers
+        self.shuffle = shuffle
 
         if testvars:
             if not os.path.exists(testvars):
                 raise Exception('--testvars file does not exist')
 
             import json
             with open(testvars) as f:
                 self.testvars = json.loads(f.read())
@@ -412,16 +414,19 @@ class MarionetteTestRunner(object):
                 testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
 
             testgroup.submit()
 
     def run_tests(self, tests):
         self.reset_test_stats()
         starttime = datetime.utcnow()
         while self.repeat >=0:
+            self.logger.info('\nROUND %d\n-------' % self.repeat)
+            if self.shuffle:
+                random.shuffle(tests)
             for test in tests:
                 self.run_test(test)
             self.repeat -= 1
         self.logger.info('\nSUMMARY\n-------')
         self.logger.info('passed: %d' % self.passed)
         self.logger.info('failed: %d' % self.failed)
         self.logger.info('todo: %d' % self.todo)
 
@@ -458,16 +463,18 @@ class MarionetteTestRunner(object):
 
         if not self.marionette:
             self.start_marionette()
 
         filepath = os.path.abspath(test)
 
         if os.path.isdir(filepath):
             for root, dirs, files in os.walk(filepath):
+                if self.shuffle:
+                    random.shuffle(files)
                 for filename in files:
                     if ((filename.startswith('test_') or filename.startswith('browser_')) and
                         (filename.endswith('.py') or filename.endswith('.js'))):
                         filepath = os.path.join(root, filename)
                         self.run_test(filepath)
                         if self.marionette.check_for_crash():
                             return
             return
@@ -500,17 +507,20 @@ class MarionetteTestRunner(object):
                               set([x['path'] for x in manifest_tests]))
             for skipped in skip_tests:
                 self.logger.info('TEST-SKIP | %s | device=%s, app=%s' %
                                  (os.path.basename(skipped),
                                   self.device,
                                   self.appName))
                 self.todo += 1
 
-            for i in manifest.get(tests=manifest_tests, **testargs):
+            target_tests = manifest.get(tests=manifest_tests, **testargs)
+            if self.shuffle:
+                random.shuffle(target_tests)
+            for i in target_tests:
                 self.run_test(i["path"])
                 if self.marionette.check_for_crash():
                     return
             return
 
         self.logger.info('TEST-START %s' % os.path.basename(test))
 
         for handler in self.test_handlers:
@@ -738,16 +748,21 @@ class MarionetteTestOptions(OptionParser
         self.add_option('--timeout',
                         dest='timeout',
                         type=int,
                         help='if a --timeout value is given, it will set the default page load timeout, search timeout and script timeout to the given value. If not passed in, it will use the default values of 30000ms for page load, 0ms for search timeout and 10000ms for script timeout')
         self.add_option('--es-server',
                         dest='es_servers',
                         action='append',
                         help='the ElasticSearch server to use for autolog submission')
+        self.add_option('--shuffle',
+                        action='store_true',
+                        dest='shuffle',
+                        default=False,
+                        help='run tests in a random order')
 
     def verify_usage(self, options, tests):
         if not tests:
             print 'must specify one or more test files, manifests, or directories'
             sys.exit(1)
 
         if not options.emulator and not options.address and not options.bin:
             print 'must specify --binary, --emulator or --address'