Bug 1313265 - Ensure unstructured logs containing TEST-UNEXPECTED-FAIL fail the job desktop_unittest.py, r=jgraham
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 27 Oct 2016 15:11:26 -0400
changeset 346788 b96b2b49be987aed5d1ffa2dcc1a549b658b458a
parent 346787 28afa97fa70dba7742a9ef15bbd9947a43711972
child 346789 9803c0158f92e49072019847e5713acd1f0ab7bd
push id10298
push userraliiev@mozilla.com
push dateMon, 14 Nov 2016 12:33:03 +0000
treeherdermozilla-aurora@7e29173b1641 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1313265
milestone52.0a1
Bug 1313265 - Ensure unstructured logs containing TEST-UNEXPECTED-FAIL fail the job desktop_unittest.py, r=jgraham I haven't seen a case of this happening in the wild, but I believe it is possible for a test to dump "TEST-UNEXPECTED-FAIL" directly to the log and mozharness using the StructuredOutputParser would not pick it up. This patch is just me being extra careful to flag potential errors like that. I'm not sure they even exist. This patch also purposefully uses substr to avoid requiring said string to show up at the beginning, which should avoid certain prefix issues we've run into in the past. MozReview-Commit-ID: 99n9YizlEDH
testing/mozharness/mozharness/mozilla/testing/errors.py
testing/mozharness/scripts/desktop_unittest.py
--- a/testing/mozharness/mozharness/mozilla/testing/errors.py
+++ b/testing/mozharness/mozharness/mozilla/testing/errors.py
@@ -101,14 +101,19 @@ TinderBoxPrintRe = {
         'retry_regex': re.compile(r'''(FAIL-SHOULD-RETRY|No space left on device|DMError|Connection to the other side was lost in a non-clean fashion|program finished with exit code 80|INFRA-ERROR|twisted.spread.pb.PBConnectionLost)''')
     },
 }
 
 TestPassed = [
     {'regex': re.compile('''(TEST-INFO|TEST-KNOWN-FAIL|TEST-PASS|INFO \| )'''), 'level': INFO},
 ]
 
+HarnessErrorList = [
+    {'substr': 'TEST-UNEXPECTED', 'level': ERROR, },
+    {'substr': 'PROCESS-CRASH', 'level': ERROR, },
+]
+
 LogcatErrorList = [
     {'substr': 'Fatal signal 11 (SIGSEGV)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'},
     {'substr': 'Fatal signal 7 (SIGBUS)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'},
     {'substr': '[JavaScript Error:', 'level': WARNING},
     {'substr': 'seccomp sandbox violation', 'level': ERROR, 'explanation': 'A content process has violated the system call sandbox (bug 790923)'},
 ]
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -24,16 +24,17 @@ sys.path.insert(1, os.path.dirname(sys.p
 from mozharness.base.errors import BaseErrorList
 from mozharness.base.log import INFO, ERROR
 from mozharness.base.script import PreScriptAction
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
 from mozharness.mozilla.buildbot import TBPL_EXCEPTION
 from mozharness.mozilla.mozbase import MozbaseMixin
 from mozharness.mozilla.structuredlog import StructuredOutputParser
+from mozharness.mozilla.testing.errors import HarnessErrorList
 from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 
 SUITE_CATEGORIES = ['gtest', 'cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill']
@@ -671,20 +672,17 @@ class DesktopUnittest(TestingMixin, Merc
                                               try_options,
                                               str_format_values=replace_dict))
                 cmd.extend(self.query_tests_args(tests_list,
                                                  try_tests,
                                                  str_format_values=replace_dict))
 
                 suite_name = suite_category + '-' + suite
                 tbpl_status, log_level = None, None
-                error_list = BaseErrorList + [{
-                    'regex': re.compile(r'''PROCESS-CRASH.*application crashed'''),
-                    'level': ERROR,
-                }]
+                error_list = BaseErrorList + HarnessErrorList
                 parser = self.get_test_output_parser(suite_category,
                                                      flavor=flavor,
                                                      config=self.config,
                                                      error_list=error_list,
                                                      log_obj=self.log_obj)
 
                 if suite_category == "reftest":
                     ref_formatter = imp.load_source(