Bug 1512352 - Improve task retry handling in test-verify; r=bc
authorGeoff Brown <gbrown@mozilla.com>
Mon, 14 Jan 2019 07:58:41 -0700
changeset 510822 500d91e30e9dcbe4b6a9a908899de5629a2f9083
parent 510821 a34a6c26b080c60356473380745af6d94ef238b7
child 510823 b8f96f4d584a2aab372db25e06ea1559193e4f40
child 510873 4d7150b71e09bed644881a9ea3f8d12a4be6adda
push id10547
push userffxbld-merge
push dateMon, 21 Jan 2019 13:03:58 +0000
treeherdermozilla-beta@24ec1916bffe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbc
bugs1512352
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1512352 - Improve task retry handling in test-verify; r=bc
testing/mozharness/scripts/android_emulator_unittest.py
testing/mozharness/scripts/android_hardware_unittest.py
testing/mozharness/scripts/desktop_unittest.py
testing/mozharness/scripts/web_platform_tests.py
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -13,16 +13,17 @@ import re
 import sys
 import subprocess
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 from mozharness.base.log import FATAL
 from mozharness.base.script import BaseScript, PreScriptAction
+from mozharness.mozilla.automation import TBPL_RETRY
 from mozharness.mozilla.mozbase import MozbaseMixin
 from mozharness.mozilla.testing.android import AndroidMixin
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
 
@@ -397,17 +398,17 @@ class AndroidEmulatorTest(TestingMixin, 
                 if (datetime.datetime.now() - self.start_time) > max_per_test_time:
                     # Running tests has run out of time. That is okay! Stop running
                     # them so that a task timeout is not triggered, and so that
                     # (partial) results are made available in a timely manner.
                     self.info("TinderboxPrint: Running tests took too long: "
                               "Not all tests were executed.<br/>")
                     # Signal per-test time exceeded, to break out of suites and
                     # suite categories loops also.
-                    return False
+                    return
 
                 final_cmd = copy.copy(cmd)
                 if len(per_test_args) > 0:
                     # in per-test mode, remove any chunk arguments from command
                     for arg in final_cmd:
                         if 'total-chunk' in arg or 'this-chunk' in arg:
                             final_cmd.remove(arg)
                 final_cmd.extend(per_test_args)
@@ -426,16 +427,19 @@ class AndroidEmulatorTest(TestingMixin, 
                                                                          previous_summary=summary)
                 parser.append_tinderboxprint_line(self.test_suite)
 
                 self.info("##### %s log ends" % self.test_suite)
 
                 if len(per_test_args) > 0:
                     self.record_status(tbpl_status, level=log_level)
                     self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
+                    if tbpl_status == TBPL_RETRY:
+                        self.info("Per-test run abandoned due to RETRY status")
+                        return
                 else:
                     self.record_status(tbpl_status, level=log_level)
                     self.log("The %s suite: %s ran with return status: %s" %
                              (suite_category, suite, tbpl_status), level=log_level)
 
 
 if __name__ == '__main__':
     test = AndroidEmulatorTest()
--- a/testing/mozharness/scripts/android_hardware_unittest.py
+++ b/testing/mozharness/scripts/android_hardware_unittest.py
@@ -13,16 +13,17 @@ import re
 import sys
 import subprocess
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 from mozharness.base.log import FATAL
 from mozharness.base.script import BaseScript, PreScriptAction
+from mozharness.mozilla.automation import TBPL_RETRY
 from mozharness.mozilla.mozbase import MozbaseMixin
 from mozharness.mozilla.testing.android import AndroidMixin
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.mozilla.testing.codecoverage import CodeCoverageMixin
 
 SUITE_DEFAULT_E10S = ['geckoview-junit', 'mochitest', 'reftest']
 SUITE_NO_E10S = ['cppunittest', 'xpcshell']
 
@@ -372,17 +373,17 @@ class AndroidHardwareTest(TestingMixin, 
                 if (datetime.datetime.now() - self.start_time) > max_per_test_time:
                     # Running tests has run out of time. That is okay! Stop running
                     # them so that a task timeout is not triggered, and so that
                     # (partial) results are made available in a timely manner.
                     self.info("TinderboxPrint: Running tests took too long: "
                               "Not all tests were executed.<br/>")
                     # Signal per-test time exceeded, to break out of suites and
                     # suite categories loops also.
-                    return False
+                    return
 
                 final_cmd = copy.copy(cmd)
                 if len(per_test_args) > 0:
                     # in per-test mode, remove any chunk arguments from command
                     for arg in final_cmd:
                         if 'total-chunk' in arg or 'this-chunk' in arg:
                             final_cmd.remove(arg)
                 final_cmd.extend(per_test_args)
@@ -401,16 +402,19 @@ class AndroidHardwareTest(TestingMixin, 
                 tbpl_status, log_level, summary = parser.evaluate_parser(0, summary)
                 parser.append_tinderboxprint_line(self.test_suite)
 
                 self.info("##### %s log ends" % self.test_suite)
 
                 if len(per_test_args) > 0:
                     self.record_status(tbpl_status, level=log_level)
                     self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
+                    if tbpl_status == TBPL_RETRY:
+                        self.info("Per-test run abandoned due to RETRY status")
+                        return
                 else:
                     self.record_status(tbpl_status, level=log_level)
                     self.log("The %s suite: %s ran with return status: %s" %
                              (suite_category, suite, tbpl_status), level=log_level)
 
 
 if __name__ == '__main__':
     test = AndroidHardwareTest()
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -23,17 +23,17 @@ from datetime import datetime, timedelta
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 from mozharness.base.errors import BaseErrorList
 from mozharness.base.log import INFO
 from mozharness.base.script import PreScriptAction
 from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.automation import TBPL_EXCEPTION
+from mozharness.mozilla.automation import TBPL_EXCEPTION, TBPL_RETRY
 from mozharness.mozilla.mozbase import MozbaseMixin
 from mozharness.mozilla.structuredlog import StructuredOutputParser
 from mozharness.mozilla.testing.errors import HarnessErrorList
 from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
@@ -938,16 +938,19 @@ class DesktopUnittest(TestingMixin, Merc
                     tbpl_status, log_level, summary = parser.evaluate_parser(return_code,
                                                                              success_codes,
                                                                              summary)
                     parser.append_tinderboxprint_line(suite_name)
 
                     self.record_status(tbpl_status, level=log_level)
                     if len(per_test_args) > 0:
                         self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
+                        if tbpl_status == TBPL_RETRY:
+                            self.info("Per-test run abandoned due to RETRY status")
+                            return False
                     else:
                         self.log("The %s suite: %s ran with return status: %s" %
                                  (suite_category, suite, tbpl_status), level=log_level)
 
             if executed_too_many_tests:
                 return False
         else:
             self.debug('There were no suites to run for %s' % suite_category)
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -14,16 +14,17 @@ from datetime import datetime, timedelta
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 import mozinfo
 
 from mozharness.base.errors import BaseErrorList
 from mozharness.base.script import PreScriptAction
 from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.automation import TBPL_RETRY
 from mozharness.mozilla.testing.android import AndroidMixin
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
 from mozharness.mozilla.testing.errors import HarnessErrorList
 
@@ -439,14 +440,17 @@ class WebPlatformTest(TestingMixin, Merc
                     self.add_per_test_coverage_report(final_env, suite, per_test_args[-1])
 
                 tbpl_status, log_level, summary = parser.evaluate_parser(return_code,
                                                                          previous_summary=summary)
                 self.record_status(tbpl_status, level=log_level)
 
                 if len(per_test_args) > 0:
                     self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
+                    if tbpl_status == TBPL_RETRY:
+                        self.info("Per-test run abandoned due to RETRY status")
+                        return
 
 
 # main {{{1
 if __name__ == '__main__':
     web_platform_tests = WebPlatformTest()
     web_platform_tests.run_and_exit()