Bug 1367745 - Make sure 'process_output' messages are logged at the proper mozharness level, r=jgraham
☠☠ backed out by b56fa748e7ec ☠ ☠
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 23 May 2017 09:05:57 -0400
changeset 585296 f19ff82014373b5c3763ad6f5bdb30350e5a1dd8
parent 585295 ad1fa17ce50ee47b0e60967d59600ddbb9672980
child 585297 2c4c92d37b9dfa2e031278654a893484a3a5dc03
push id61093
push userdgottwald@mozilla.com
push dateFri, 26 May 2017 20:16:26 +0000
reviewersjgraham
bugs1367745
milestone55.0a1
Bug 1367745 - Make sure 'process_output' messages are logged at the proper mozharness level, r=jgraham In addition to saving the log level of 'process_output' messages, this will also start passing 'log' messages through the error lists. This means mozharness will start using 'log' errors when determining the tbpl_status and worst_log_level. MozReview-Commit-ID: CZnH6aI1Wo0
testing/mozharness/mozharness/base/log.py
testing/mozharness/mozharness/mozilla/structuredlog.py
--- a/testing/mozharness/mozharness/base/log.py
+++ b/testing/mozharness/mozharness/base/log.py
@@ -292,30 +292,31 @@ class OutputParser(LogMixin):
 
     def parse_single_line(self, line):
         """ parse a console output line and check if it matches one in `error_list`,
         if so then log it according to `log_output`.
 
         Args:
             line (str): command line output to parse.
         """
+        log_level = INFO
         for error_check in self.error_list:
             # TODO buffer for context_lines.
             match = False
             if 'substr' in error_check:
                 if error_check['substr'] in line:
                     match = True
             elif 'regex' in error_check:
                 if error_check['regex'].search(line):
                     match = True
             else:
                 self.warning("error_list: 'substr' and 'regex' not in %s" %
                              error_check)
             if match:
-                log_level = error_check.get('level', INFO)
+                log_level = error_check.get('level', log_level)
                 if self.log_output:
                     message = ' %s' % line
                     if error_check.get('explanation'):
                         message += '\n %s' % error_check['explanation']
                     if error_check.get('summary'):
                         self.add_summary(message, level=log_level)
                     else:
                         self.log(message, level=log_level)
@@ -323,17 +324,19 @@ class OutputParser(LogMixin):
                     self.num_errors += 1
                 if log_level == WARNING:
                     self.num_warnings += 1
                 self.worst_log_level = self.worst_level(log_level,
                                                         self.worst_log_level)
                 break
         else:
             if self.log_output:
-                self.info(' %s' % line)
+                self.log(' %s' % line, level=log_level)
+
+        return log_level
 
     def add_lines(self, output):
         """ process a string or list of strings, decode them to utf-8,strip
         them of any trailing whitespaces and parse them using `parse_single_line`
 
         strings consisting only of whitespaces are ignored.
 
         Args:
--- a/testing/mozharness/mozharness/mozilla/structuredlog.py
+++ b/testing/mozharness/mozharness/mozilla/structuredlog.py
@@ -44,17 +44,17 @@ class StructuredOutputParser(OutputParse
             import mozlog
         except ImportError:
             self.fatal("A script class using structured logging must inherit "
                        "from the MozbaseMixin to ensure that mozlog is available.")
         return mozlog
 
     def _handle_unstructured_output(self, line, log_output=True):
         self.log_output = log_output
-        super(StructuredOutputParser, self).parse_single_line(line)
+        return super(StructuredOutputParser, self).parse_single_line(line)
 
     def parse_single_line(self, line):
         """Parses a line of log output from the child process and passes
         it to mozlog to update the overall status of the run.
         Re-emits the logged line in human-readable format.
         """
         level = INFO
         tbpl_level = TBPL_SUCCESS
@@ -75,23 +75,29 @@ class StructuredOutputParser(OutputParse
                 self.update_levels(TBPL_FAILURE, log.CRITICAL)
             else:
                 self._handle_unstructured_output(line)
             return
 
         self.handler(data)
 
         action = data["action"]
-        if action == "process_output":
-            # Run process output through the error lists, but make sure the super parser
-            # doesn't print them to stdout (they should go through the log formatter).
-            self._handle_unstructured_output(data['data'], log_output=False)
+        if action in ('log', 'process_output'):
+            # Run log and process_output actions through the error lists, but make sure
+            # the super parser doesn't print them to stdout (they should go through the
+            # log formatter).
+            if 'message' in data:
+                message = data['message']
+            else:
+                message = data['data']
+            error_level = self._handle_unstructured_output(message, log_output=False)
 
-        if action == "log":
-            level = getattr(log, data["level"].upper())
+            if 'level' in data:
+                level = getattr(log, data['level'].upper())
+            level = self.worst_level(error_level, level)
 
         log_data = self.formatter(data)
         if log_data is not None:
             self.log(log_data, level=level)
             self.update_levels(tbpl_level, level)
 
     def evaluate_parser(self, return_code, success_codes=None):
         success_codes = success_codes or [0]