[mq]: seta_tests.patch draft
authorJoel Maher <jmaher@mozilla.com>
Thu, 28 Jun 2018 15:05:17 -0400
changeset 1551689 845e2b081bd8
parent 1551688 9f556dd25aca
child 1551690 9afbcd3852b5
push id281347
push userjmaher@mozilla.com
push dateThu, 28 Jun 2018 19:09:55 +0000
treeherdertry@beb4cd121c5f [default view] [failures only]
milestone63.0a1
[mq]: seta_tests.patch
layout/tools/reftest/runreftest.py
taskcluster/taskgraph/util/perfile.py
testing/mochitest/runtests.py
testing/mozharness/scripts/desktop_unittest.py
testing/mozharness/scripts/web_platform_tests.py
testing/web-platform/tests/tools/wptrunner/wptrunner/stability.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
testing/xpcshell/runxpcshelltests.py
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -459,17 +459,17 @@ class RefTest(object):
         failures.
         """
 
         self._populate_logger(options)
 
         # Number of times to repeat test(s) when running with --repeat
         VERIFY_REPEAT = 10
         # Number of times to repeat test(s) when running test in separate browser
-        VERIFY_REPEAT_SINGLE_BROWSER = 5
+        VERIFY_REPEAT_SINGLE_BROWSER = 1
 
         def step1():
             stepOptions = copy.deepcopy(options)
             stepOptions.repeat = VERIFY_REPEAT
             stepOptions.runUntilFailure = True
             result = self.runTests(tests, stepOptions)
             return result
 
@@ -506,16 +506,22 @@ class RefTest(object):
              step2),
             ("3. Run each test %d times in one browser, in chaos mode." % VERIFY_REPEAT,
              step3),
             ("4. Run each test %d times in a new browser each time, in chaos mode." %
              VERIFY_REPEAT_SINGLE_BROWSER,
              step4),
         ]
 
+        steps = [
+            ("1. Run each test %d times in a new browser each time." %
+             VERIFY_REPEAT_SINGLE_BROWSER,
+             step2),
+        ]
+
         stepResults = {}
         for (descr, step) in steps:
             stepResults[descr] = "not run / incomplete"
 
         startTime = datetime.now()
         maxTime = timedelta(seconds=options.verify_max_time)
         finalResult = "PASSED"
         for (descr, step) in steps:
--- a/taskcluster/taskgraph/util/perfile.py
+++ b/taskcluster/taskgraph/util/perfile.py
@@ -17,17 +17,17 @@ from subprocess import CalledProcessErro
 from taskgraph import files_changed
 from .. import GECKO
 
 logger = logging.getLogger(__name__)
 
 
 @memoize
 def perfile_number_of_chunks(try_task_config, head_repository, head_rev, type):
-    tests_per_chunk = 10.0
+    tests_per_chunk = 30.0
     if type.startswith('test-coverage'):
         tests_per_chunk = 30.0
 
     if type.startswith('test-verify-wpt') or type.startswith('test-coverage-wpt'):
         file_patterns = ['testing/web-platform/tests/**',
                          'testing/web-platform/mozilla/tests/**']
     elif type.startswith('test-verify-gpu') or type.startswith('test-coverage-gpu'):
         file_patterns = ['**/*webgl*/**/test_*',
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -2439,17 +2439,17 @@ toolbar#nav-bar {
         Support --verify mode: Run test(s) many times in a variety of
         configurations/environments in an effort to find intermittent
         failures.
         """
 
         # Number of times to repeat test(s) when running with --repeat
         VERIFY_REPEAT = 10
         # Number of times to repeat test(s) when running test in
-        VERIFY_REPEAT_SINGLE_BROWSER = 5
+        VERIFY_REPEAT_SINGLE_BROWSER = 1
 
         def step1():
             stepOptions = copy.deepcopy(options)
             stepOptions.repeat = VERIFY_REPEAT
             stepOptions.keep_open = False
             stepOptions.runUntilFailure = True
             stepOptions.profilePath = None
             self.urlOpts = []
@@ -2508,16 +2508,22 @@ toolbar#nav-bar {
             ("3. Run each test %d times in one browser, in chaos mode." %
              VERIFY_REPEAT,
              step3),
             ("4. Run each test %d times in a new browser each time, "
              "in chaos mode." % VERIFY_REPEAT_SINGLE_BROWSER,
              step4),
         ]
 
+        steps = [
+            ("1. Run each test %d times in a new browser each time." %
+             VERIFY_REPEAT_SINGLE_BROWSER,
+             step2),
+        ]
+
         stepResults = {}
         for (descr, step) in steps:
             stepResults[descr] = "not run / incomplete"
 
         startTime = datetime.now()
         maxTime = timedelta(seconds=options.verify_max_time)
         finalResult = "PASSED"
         for (descr, step) in steps:
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -764,17 +764,17 @@ class DesktopUnittest(TestingMixin, Merc
     def _run_category_suites(self, suite_category):
         """run suite(s) to a specific category"""
         dirs = self.query_abs_dirs()
         suites = self._query_specified_suites(suite_category)
         abs_app_dir = self.query_abs_app_dir()
         abs_res_dir = self.query_abs_res_dir()
 
         max_per_test_time = timedelta(minutes=60)
-        max_per_test_tests = 10
+        max_per_test_tests = 30
         if self.per_test_coverage:
             max_per_test_tests = 30
         executed_tests = 0
         executed_too_many_tests = False
 
         if suites:
             self.info('#### Running %s suites' % suite_category)
             for suite in suites:
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -312,17 +312,17 @@ class WebPlatformTest(TestingMixin, Merc
             env['STYLO_THREADS'] = '1'
         else:
             env['STYLO_THREADS'] = '4'
 
         env = self.query_env(partial_env=env, log_level=INFO)
 
         start_time = datetime.now()
         max_per_test_time = timedelta(minutes=60)
-        max_per_test_tests = 10
+        max_per_test_tests = 30
         if self.per_test_coverage:
             max_per_test_tests = 30
         executed_tests = 0
         executed_too_many_tests = False
 
         if self.per_test_coverage or self.verify_enabled:
             suites = self.query_per_test_category_suites(None, None)
             if "wdspec" in suites:
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/stability.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/stability.py
@@ -238,17 +238,17 @@ def write_summary(logger, step_results, 
         log = logger.warning
     else:
         log = logger.error
     log('::: Test verification %s' % final_result)
 
     logger.info(':::')
 
 
-def check_stability(logger, repeat_loop=10, repeat_restart=5, chaos_mode=True, max_time=None,
+def check_stability(logger, repeat_loop=1, repeat_restart=0, chaos_mode=False, max_time=None,
                     output_results=True, **kwargs):
     kwargs_extras = [{}]
     if chaos_mode and kwargs["product"] == "firefox":
         kwargs_extras.append({"chaos_mode_flags": 3})
 
     steps = get_steps(logger, repeat_loop, repeat_restart, kwargs_extras)
 
     start_time = datetime.now()
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
@@ -300,17 +300,18 @@ def run_tests(config, test_paths, produc
 
 def check_stability(**kwargs):
     import stability
     if kwargs["stability"]:
         logger.warning("--stability is deprecated; please use --verify instead!")
         kwargs['verify_max_time'] = None
         kwargs['verify_chaos_mode'] = False
         kwargs['verify_repeat_loop'] = 0
-        kwargs['verify_repeat_restart'] = 10 if kwargs['repeat'] == 1 else kwargs['repeat']
+#        kwargs['verify_repeat_restart'] = 10 if kwargs['repeat'] == 1 else kwargs['repeat']
+        kwargs['verify_repeat_restart'] = 1
         kwargs['verify_output_results'] = True
 
     return stability.check_stability(logger,
                                      max_time=kwargs['verify_max_time'],
                                      chaos_mode=kwargs['verify_chaos_mode'],
                                      repeat_loop=kwargs['verify_repeat_loop'],
                                      repeat_restart=kwargs['verify_repeat_restart'],
                                      output_results=kwargs['verify_output_results'],
--- a/testing/xpcshell/runxpcshelltests.py
+++ b/testing/xpcshell/runxpcshelltests.py
@@ -1151,17 +1151,17 @@ class XPCShellTests(object):
     def runTests(self, options, testClass=XPCShellTestThread, mobileArgs=None):
         """
           Run xpcshell tests.
         """
 
         global gotSIGINT
 
         # Number of times to repeat test(s) in --verify mode
-        VERIFY_REPEAT = 10
+        VERIFY_REPEAT = 1
 
         if isinstance(options, Namespace):
             options = vars(options)
 
         # Try to guess modules directory.
         # This somewhat grotesque hack allows the buildbot machines to find the
         # modules directory without having to configure the buildbot hosts. This
         # code path should never be executed in local runs because the build system
@@ -1402,16 +1402,20 @@ class XPCShellTests(object):
                 return status
 
             steps = [
                 ("1. Run each test %d times, sequentially." % VERIFY_REPEAT,
                  step1),
                 ("2. Run each test %d times, sequentially, in chaos mode." % VERIFY_REPEAT,
                  step2),
             ]
+            steps = [
+                ("1. Run each test %d times, sequentially." % VERIFY_REPEAT,
+                 step1),
+            ]
             startTime = datetime.now()
             maxTime = timedelta(seconds=options['verifyMaxTime'])
             for test_object in self.alltests:
                 stepResults = {}
                 for (descr, step) in steps:
                     stepResults[descr] = "not run / incomplete"
                 finalResult = "PASSED"
                 for (descr, step) in steps: