Bug 1561309 - add --repeat command line argument for mozharness reftest and mochitest categories, r=ahal.
authorBob Clary <bclary@bclary.com>
Fri, 05 Jul 2019 18:39:30 +0000
changeset 544330 1373ded9ebd53354a69d8637f4c34b3abe89df19
parent 544329 7d95f38f8dada3b12f299d9fe36822e8dacb6d1e
child 544331 4ea815152fd7e5d353cfdaebe21ae6840dc69900
push id2131
push userffxbld-merge
push dateMon, 26 Aug 2019 18:30:20 +0000
treeherdermozilla-release@b19ffb3ca153 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1561309
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1561309 - add --repeat command line argument for mozharness reftest and mochitest categories, r=ahal. Differential Revision: https://phabricator.services.mozilla.com/D36011
testing/mozharness/scripts/android_emulator_unittest.py
testing/mozharness/scripts/android_hardware_unittest.py
testing/mozharness/scripts/desktop_unittest.py
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -10,29 +10,30 @@ import datetime
 import json
 import os
 import sys
 import subprocess
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
-from mozharness.base.log import FATAL
+from mozharness.base.log import WARNING, FATAL
 from mozharness.base.script import BaseScript, PreScriptAction
 from mozharness.mozilla.automation import TBPL_RETRY
 from mozharness.mozilla.mozbase import MozbaseMixin
 from mozharness.mozilla.testing.android import AndroidMixin
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
 
 SUITE_DEFAULT_E10S = ['geckoview-junit', 'mochitest', 'reftest']
 SUITE_NO_E10S = ['cppunittest', 'geckoview-junit', 'xpcshell']
+SUITE_REPEATABLE = ['mochitest', 'reftest']
 
 
 class AndroidEmulatorTest(TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin,
                           AndroidMixin):
     """
        A mozharness script for Android functional tests (like mochitests and reftests)
        run on an Android emulator. This script starts and manages an Android emulator
        for the duration of the required tests. This is like desktop_unittest.py, but
@@ -88,16 +89,25 @@ class AndroidEmulatorTest(TestingMixin, 
          }
     ], [
         ['--enable-webrender'],
         {"action": "store_true",
          "dest": "enable_webrender",
          "default": False,
          "help": "Run with WebRender enabled.",
          }
+    ], [
+        ['--repeat'],
+        {"action": "store",
+         "type": "int",
+         "dest": "repeat",
+         "default": 0,
+         "help": "Repeat the tests the given number of times. Supported "
+                 "by mochitest, reftest, crashtest, ignored otherwise."
+         }
     ]] + copy.deepcopy(testing_config_options) + \
         copy.deepcopy(code_coverage_config_options)
 
     def __init__(self, require_config_file=False):
         super(AndroidEmulatorTest, self).__init__(
             config_options=self.config_options,
             all_actions=['clobber',
                          'setup-avds',
@@ -265,16 +275,21 @@ class AndroidEmulatorTest(TestingMixin, 
             category = 'reftest'
         else:
             category = self.test_suite
         if category not in SUITE_NO_E10S:
             if category in SUITE_DEFAULT_E10S and not self.e10s:
                 cmd.extend(['--disable-e10s'])
             elif category not in SUITE_DEFAULT_E10S and self.e10s:
                 cmd.extend(['--e10s'])
+        if c.get('repeat'):
+            if category in SUITE_REPEATABLE:
+                cmd.extend(["--repeat=%s" % c.get('repeat')])
+            else:
+                self.log("--repeat not supported in {}".format(category), level=WARNING)
 
         if not (self.verify_enabled or self.per_test_coverage):
             if user_paths:
                 cmd.extend(user_paths)
             elif not (self.verify_enabled or self.per_test_coverage):
                 if self.this_chunk is not None:
                     cmd.extend(['--this-chunk', self.this_chunk])
                 if self.total_chunks is not None:
--- a/testing/mozharness/scripts/android_hardware_unittest.py
+++ b/testing/mozharness/scripts/android_hardware_unittest.py
@@ -10,26 +10,27 @@ import datetime
 import json
 import os
 import sys
 import subprocess
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
-from mozharness.base.log import FATAL
+from mozharness.base.log import WARNING, FATAL
 from mozharness.base.script import BaseScript, PreScriptAction
 from mozharness.mozilla.automation import TBPL_RETRY
 from mozharness.mozilla.mozbase import MozbaseMixin
 from mozharness.mozilla.testing.android import AndroidMixin
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.mozilla.testing.codecoverage import CodeCoverageMixin
 
 SUITE_DEFAULT_E10S = ['geckoview-junit', 'mochitest', 'reftest']
 SUITE_NO_E10S = ['cppunittest', 'xpcshell']
+SUITE_REPEATABLE = ['mochitest', 'reftest']
 
 
 class AndroidHardwareTest(TestingMixin, BaseScript, MozbaseMixin,
                           CodeCoverageMixin, AndroidMixin):
     config_options = [[
         ["--test-suite"],
         {"action": "store",
          "dest": "test_suite",
@@ -79,16 +80,25 @@ class AndroidHardwareTest(TestingMixin, 
          }
     ], [
         ['--enable-webrender'],
         {"action": "store_true",
          "dest": "enable_webrender",
          "default": False,
          "help": "Run with WebRender enabled.",
          }
+    ], [
+        ['--repeat'],
+        {"action": "store",
+         "type": "int",
+         "dest": "repeat",
+         "default": 0,
+         "help": "Repeat the tests the given number of times. Supported "
+                 "by mochitest, reftest, crashtest, ignored otherwise."
+         }
     ]] + copy.deepcopy(testing_config_options)
 
     def __init__(self, require_config_file=False):
         super(AndroidHardwareTest, self).__init__(
             config_options=self.config_options,
             all_actions=['clobber',
                          'download-and-extract',
                          'create-virtualenv',
@@ -249,16 +259,21 @@ class AndroidHardwareTest(TestingMixin, 
             category = 'reftest'
         else:
             category = self.test_suite
         if category not in SUITE_NO_E10S:
             if category in SUITE_DEFAULT_E10S and not self.e10s:
                 cmd.extend(['--disable-e10s'])
             elif category not in SUITE_DEFAULT_E10S and self.e10s:
                 cmd.extend(['--e10s'])
+        if c.get('repeat'):
+            if category in SUITE_REPEATABLE:
+                cmd.extend(["--repeat=%s" % c.get('repeat')])
+            else:
+                self.log("--repeat not supported in {}".format(category), level=WARNING)
 
         # Only enable WebRender if the flag is enabled. All downstream harnesses
         # are expected to force-disable WebRender if not explicitly enabled,
         # so that we don't have it accidentally getting enabled because the
         # underlying hardware running the test becomes part of the WR-qualified
         # set.
         if self.enable_webrender:
             cmd.extend(['--enable-webrender'])
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -22,17 +22,17 @@ import platform
 
 from datetime import datetime, timedelta
 
 # load modules from parent dir
 here = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(1, os.path.dirname(here))
 
 from mozharness.base.errors import BaseErrorList
-from mozharness.base.log import INFO
+from mozharness.base.log import INFO, WARNING
 from mozharness.base.script import PreScriptAction
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.automation import TBPL_EXCEPTION, TBPL_RETRY
 from mozharness.mozilla.mozbase import MozbaseMixin
 from mozharness.mozilla.structuredlog import StructuredOutputParser
 from mozharness.mozilla.testing.errors import HarnessErrorList
 from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
 from mozharness.mozilla.testing.codecoverage import (
@@ -40,16 +40,17 @@ from mozharness.mozilla.testing.codecove
     code_coverage_config_options
 )
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 
 SUITE_CATEGORIES = ['gtest', 'cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell',
                     'mozmill']
 SUITE_DEFAULT_E10S = ['mochitest', 'reftest']
 SUITE_NO_E10S = ['xpcshell']
+SUITE_REPEATABLE = ['mochitest', 'reftest']
 
 
 # DesktopUnittest {{{1
 class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin,
                       CodeCoverageMixin):
     config_options = [
         [['--mochitest-suite', ], {
             "action": "extend",
@@ -170,16 +171,24 @@ class DesktopUnittest(TestingMixin, Merc
          ],
         [["--setpref"], {
             "action": "append",
             "metavar": "PREF=VALUE",
             "dest": "extra_prefs",
             "default": [],
             "help": "Defines an extra user preference."}
          ],
+        [['--repeat', ], {
+            "action": "store",
+            "type": "int",
+            "dest": "repeat",
+            "default": 0,
+            "help": "Repeat the tests the given number of times. Supported "
+                    "by mochitest, reftest, crashtest, ignored otherwise."}
+         ],
     ] + copy.deepcopy(testing_config_options) + \
         copy.deepcopy(code_coverage_config_options)
 
     def __init__(self, require_config_file=True):
         # abs_dirs defined already in BaseScript but is here to make pylint happy
         self.abs_dirs = None
         super(DesktopUnittest, self).__init__(
             config_options=self.config_options,
@@ -414,16 +423,21 @@ class DesktopUnittest(TestingMixin, Merc
             if self.symbols_path:
                 str_format_values['symbols_path'] = self.symbols_path
 
             if suite_category not in SUITE_NO_E10S:
                 if suite_category in SUITE_DEFAULT_E10S and not c['e10s']:
                     base_cmd.append('--disable-e10s')
                 elif suite_category not in SUITE_DEFAULT_E10S and c['e10s']:
                     base_cmd.append('--e10s')
+            if c.get('repeat'):
+                if suite_category in SUITE_REPEATABLE:
+                    base_cmd.extend(["--repeat=%s" % c.get('repeat')])
+                else:
+                    self.log("--repeat not supported in {}".format(suite_category), level=WARNING)
 
             # Ignore chunking if we have user specified test paths
             if not (self.verify_enabled or self.per_test_coverage):
                 test_paths = self._get_mozharness_test_paths(suite_category, suite)
                 if test_paths:
                     base_cmd.extend(test_paths)
                 elif c.get('total_chunks') and c.get('this_chunk'):
                     base_cmd.extend(['--total-chunks', c['total_chunks'],