Bug 1119830 - Mozharness should use return_code in its tbpl_status determination, r=chmanchester
--- a/mozharness/mozilla/structuredlog.py
+++ b/mozharness/mozilla/structuredlog.py
@@ -80,17 +80,18 @@ class StructuredOutputParser(OutputParse
action = data["action"]
if action == "log":
level = getattr(log, data["level"].upper())
self.log(self.formatter(data), level=level)
self.update_levels(tbpl_level, level)
- def evaluate_parser(self, return_code):
+ def evaluate_parser(self, return_code, success_codes=None):
+ success_codes = success_codes or [0]
summary = self.handler.summarize()
fail_pair = TBPL_WARNING, WARNING
error_pair = TBPL_FAILURE, ERROR
# These are warning/orange statuses.
failure_conditions = [
sum(summary.unexpected_statuses.values()) > 0,
@@ -118,17 +119,17 @@ class StructuredOutputParser(OutputParse
# If a superclass was used to detect errors with a regex based output parser,
# this will be reflected in the status here.
if self.num_errors:
self.update_levels(*error_pair)
# Harnesses typically return non-zero on test failure, so don't promote
# to error if we already have a failing status.
- if return_code != 0 and self.tbpl_status == TBPL_SUCCESS:
+ if return_code not in success_codes and self.tbpl_status == TBPL_SUCCESS:
self.update_levels(*error_pair)
return self.tbpl_status, self.worst_log_level
def update_levels(self, tbpl_level, log_level):
self.worst_log_level = self.worst_level(log_level, self.worst_log_level)
self.tbpl_status = self.worst_level(tbpl_level, self.tbpl_status,
levels=TBPL_WORST_LEVEL_TUPLE)--- a/mozharness/mozilla/testing/unittest.py
+++ b/mozharness/mozilla/testing/unittest.py
@@ -151,17 +151,19 @@ class DesktopUnittestOutputParser(Output
if self.harness_retry_re.search(line):
self.critical(' %s' % line)
self.worst_log_level = self.worst_level(CRITICAL, self.worst_log_level)
self.tbpl_status = self.worst_level(TBPL_RETRY, self.tbpl_status,
levels=TBPL_WORST_LEVEL_TUPLE)
return # skip base parse_single_line
super(DesktopUnittestOutputParser, self).parse_single_line(line)
- def evaluate_parser(self, return_code):
+ def evaluate_parser(self, return_code, success_codes=None):
+ success_codes = success_codes or [0]
+
if self.num_errors: # mozharness ran into a script error
self.tbpl_status = self.worst_level(TBPL_FAILURE, self.tbpl_status,
levels=TBPL_WORST_LEVEL_TUPLE)
# I have to put this outside of parse_single_line because this checks not
# only if fail_count was more then 0 but also if fail_count is still -1
# (no fail summary line was found)
if self.fail_count != 0:
@@ -174,16 +176,20 @@ class DesktopUnittestOutputParser(Output
(self.known_fail_count is None or self.known_fail_count <= 0):
self.error('No tests run or test summary not found')
self.worst_log_level = self.worst_level(WARNING,
self.worst_log_level)
self.tbpl_status = self.worst_level(TBPL_WARNING,
self.tbpl_status,
levels=TBPL_WORST_LEVEL_TUPLE)
+ if return_code not in success_codes:
+ self.tbpl_status = self.worst_level(TBPL_FAILURE, self.tbpl_status,
+ levels=TBPL_WORST_LEVEL_TUPLE)
+
# we can trust in parser.worst_log_level in either case
return (self.tbpl_status, self.worst_log_level)
def append_tinderboxprint_line(self, suite_name):
# We are duplicating a condition (fail_count) from evaluate_parser and
# parse parse_single_line but at little cost since we are not parsing
# the log more then once. I figured this method should stay isolated as
# it is only here for tbpl highlighted summaries and is not part of--- a/scripts/android_panda.py
+++ b/scripts/android_panda.py
@@ -260,22 +260,22 @@ class PandaTest(TestingMixin, MercurialS
if env.has_key('PYTHONPATH'):
del env['PYTHONPATH']
parser = self.get_test_output_parser(suite_category,
config=self.config,
error_list=error_list,
log_obj=self.log_obj)
- self.run_command(cmd,
- cwd=dirs['abs_test_install_dir'],
- env=env,
- output_parser=parser)
+ return_code = self.run_command(cmd,
+ cwd=dirs['abs_test_install_dir'],
+ env=env,
+ output_parser=parser)
- tbpl_status, log_level = parser.evaluate_parser(0)
+ tbpl_status, log_level = parser.evaluate_parser(return_code)
if tbpl_status != TBPL_SUCCESS:
self.info("Output logcat...")
try:
lines = self.get_logcat()
self.info("*** STARTING LOGCAT ***")
for l in lines:
self.info(l)--- a/scripts/b2g_desktop_unittest.py
+++ b/scripts/b2g_desktop_unittest.py
@@ -260,22 +260,23 @@ class B2GDesktopTest(BlobUploadMixin, Te
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
env = self.query_env(partial_env=env)
parser = self.get_test_output_parser(suite_name,
config=self.config,
log_obj=self.log_obj,
error_list=error_list)
- self.run_command(cmd, cwd=cwd, env=env,
- output_timeout=1000,
- output_parser=parser,
- success_codes=success_codes)
+ return_code = self.run_command(cmd, cwd=cwd, env=env,
+ output_timeout=1000,
+ output_parser=parser,
+ success_codes=success_codes)
- tbpl_status, log_level = parser.evaluate_parser(0)
+ tbpl_status, log_level = parser.evaluate_parser(return_code,
+ success_codes=success_codes)
parser.append_tinderboxprint_line(suite_name)
self.buildbot_status(tbpl_status, level=log_level)
self.log("The %s suite: %s ran with return status: %s" %
(suite_name, suite, tbpl_status), level=log_level)
if __name__ == '__main__':
desktopTest = B2GDesktopTest()--- a/scripts/b2g_emulator_unittest.py
+++ b/scripts/b2g_emulator_unittest.py
@@ -303,17 +303,17 @@ class B2GEmulatorTest(TestingMixin, Tool
missing_key = False
options = self.tree_config["suite_definitions"][suite]["options"]
else:
suite_options = '%s_options' % suite
if suite_options in self.tree_config:
missing_key = False
options = self.tree_config[suite_options]
- if missing_key:
+ if missing_key:
self.fatal("Key '%s' not defined in the in-tree config! Please add it to '%s'." \
"See bug 981030 for more details." % (suite,
os.path.join('gecko', 'testing', self.config['in_tree_config'])))
if options:
for option in options:
option = option % str_format_values
if not option.endswith('None'):
@@ -382,20 +382,23 @@ class B2GEmulatorTest(TestingMixin, Tool
# expedites things; please file bugs to merge if that happens.
suite_name = [x for x in self.test_suites if x in self.config['test_suite']][0]
if self.config.get('this_chunk'):
suite = '%s-%s' % (suite_name, self.config['this_chunk'])
else:
suite = suite_name
- # bug 773703
success_codes = None
if suite_name == 'xpcshell':
+ # bug 773703
success_codes = [0, 1]
+ elif suite_name == 'mochitest' and int(self.config['this_chunk']) == 11:
+ # bug 1120580
+ success_codes = [0, 247]
if suite_name == 'cppunittest':
# check if separate test package required
if not os.path.isdir(dirs['abs_cppunittest_dir']):
self._download_unzip(self.test_url.replace('tests', 'tests.cppunit'), dirs['abs_test_install_dir'])
env = {}
if self.query_minidump_stackwalk():
@@ -404,29 +407,30 @@ class B2GEmulatorTest(TestingMixin, Tool
if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
env = self.query_env(partial_env=env)
parser = self.get_test_output_parser(suite_name,
config=self.config,
log_obj=self.log_obj,
error_list=error_list)
- self.run_command(cmd, cwd=cwd, env=env,
- output_timeout=1000,
- output_parser=parser,
- success_codes=success_codes)
+ return_code = self.run_command(cmd, cwd=cwd, env=env,
+ output_timeout=1000,
+ output_parser=parser,
+ success_codes=success_codes)
logcat = os.path.join(dirs['abs_work_dir'], 'emulator-5554.log')
qemu = os.path.join(dirs['abs_work_dir'], 'qemu.log')
if os.path.isfile(qemu):
self.copyfile(qemu, os.path.join(env['MOZ_UPLOAD_DIR'],
os.path.basename(qemu)))
- tbpl_status, log_level = parser.evaluate_parser(0)
+ tbpl_status, log_level = parser.evaluate_parser(return_code,
+ success_codes=success_codes)
if os.path.isfile(logcat):
if tbpl_status != TBPL_SUCCESS:
# On failure, dump logcat, check if the emulator is still
# running, and if it is still accessible via adb.
self.info('dumping logcat')
self.run_command(['cat', logcat], error_list=LogcatErrorList)
--- a/scripts/desktop_unittest.py
+++ b/scripts/desktop_unittest.py
@@ -540,17 +540,25 @@ class DesktopUnittest(TestingMixin, Merc
# mochitest, reftest, and xpcshell suites do not return
# appropriate return codes. Therefore, we must parse the output
# to determine what the tbpl_status and worst_log_level must
# be. We do this by:
# 1) checking to see if our mozharness script ran into any
# errors itself with 'num_errors' <- OutputParser
# 2) if num_errors is 0 then we look in the subclassed 'parser'
# findings for harness/suite errors <- DesktopUnittestOutputParser
- tbpl_status, log_level = parser.evaluate_parser(0)
+ # 3) checking to see if the return code is in success_codes
+
+ success_codes = None
+ if self._is_windows():
+ # bug 1120644
+ success_codes = [0, 1]
+
+ tbpl_status, log_level = parser.evaluate_parser(return_code,
+ success_codes=success_codes)
parser.append_tinderboxprint_line(suite_name)
self.buildbot_status(tbpl_status, level=log_level)
self.log("The %s suite: %s ran with return status: %s" %
(suite_category, suite, tbpl_status), level=log_level)
else:
self.debug('There were no suites to run for %s' % suite_category)
