Bug 1367745 - Make sure 'process_output' messages are logged at the proper mozharness level, r=jgraham
☠☠ backed out by b56fa748e7ec ☠ ☠
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 23 May 2017 09:05:57 -0400
changeset 360821 f19ff82014373b5c3763ad6f5bdb30350e5a1dd8
parent 360820 ad1fa17ce50ee47b0e60967d59600ddbb9672980
child 360822 2c4c92d37b9dfa2e031278654a893484a3a5dc03
push id31902
push userryanvm@gmail.com
push dateFri, 26 May 2017 19:43:26 +0000
treeherdermozilla-central@ba1a33add29d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1367745
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1367745 - Make sure 'process_output' messages are logged at the proper mozharness level, r=jgraham In addition to saving the log level of 'process_output' messages, this will also start passing 'log' messages through the error lists. This means mozharness will start using 'log' errors when determining the tbpl_status and worst_log_level. MozReview-Commit-ID: CZnH6aI1Wo0
testing/mozharness/mozharness/base/log.py
testing/mozharness/mozharness/mozilla/structuredlog.py
--- a/testing/mozharness/mozharness/base/log.py
+++ b/testing/mozharness/mozharness/base/log.py
@@ -292,30 +292,31 @@ class OutputParser(LogMixin):
 
     def parse_single_line(self, line):
         """ parse a console output line and check if it matches one in `error_list`,
         if so then log it according to `log_output`.
 
         Args:
             line (str): command line output to parse.
         """
+        log_level = INFO
         for error_check in self.error_list:
             # TODO buffer for context_lines.
             match = False
             if 'substr' in error_check:
                 if error_check['substr'] in line:
                     match = True
             elif 'regex' in error_check:
                 if error_check['regex'].search(line):
                     match = True
             else:
                 self.warning("error_list: 'substr' and 'regex' not in %s" %
                              error_check)
             if match:
-                log_level = error_check.get('level', INFO)
+                log_level = error_check.get('level', log_level)
                 if self.log_output:
                     message = ' %s' % line
                     if error_check.get('explanation'):
                         message += '\n %s' % error_check['explanation']
                     if error_check.get('summary'):
                         self.add_summary(message, level=log_level)
                     else:
                         self.log(message, level=log_level)
@@ -323,17 +324,19 @@ class OutputParser(LogMixin):
                     self.num_errors += 1
                 if log_level == WARNING:
                     self.num_warnings += 1
                 self.worst_log_level = self.worst_level(log_level,
                                                         self.worst_log_level)
                 break
         else:
             if self.log_output:
-                self.info(' %s' % line)
+                self.log(' %s' % line, level=log_level)
+
+        return log_level
 
     def add_lines(self, output):
         """ process a string or list of strings, decode them to utf-8,strip
         them of any trailing whitespaces and parse them using `parse_single_line`
 
         strings consisting only of whitespaces are ignored.
 
         Args:
--- a/testing/mozharness/mozharness/mozilla/structuredlog.py
+++ b/testing/mozharness/mozharness/mozilla/structuredlog.py
@@ -44,17 +44,17 @@ class StructuredOutputParser(OutputParse
             import mozlog
         except ImportError:
             self.fatal("A script class using structured logging must inherit "
                        "from the MozbaseMixin to ensure that mozlog is available.")
         return mozlog
 
     def _handle_unstructured_output(self, line, log_output=True):
         self.log_output = log_output
-        super(StructuredOutputParser, self).parse_single_line(line)
+        return super(StructuredOutputParser, self).parse_single_line(line)
 
     def parse_single_line(self, line):
         """Parses a line of log output from the child process and passes
         it to mozlog to update the overall status of the run.
         Re-emits the logged line in human-readable format.
         """
         level = INFO
         tbpl_level = TBPL_SUCCESS
@@ -75,23 +75,29 @@ class StructuredOutputParser(OutputParse
                 self.update_levels(TBPL_FAILURE, log.CRITICAL)
             else:
                 self._handle_unstructured_output(line)
             return
 
         self.handler(data)
 
         action = data["action"]
-        if action == "process_output":
-            # Run process output through the error lists, but make sure the super parser
-            # doesn't print them to stdout (they should go through the log formatter).
-            self._handle_unstructured_output(data['data'], log_output=False)
+        if action in ('log', 'process_output'):
+            # Run log and process_output actions through the error lists, but make sure
+            # the super parser doesn't print them to stdout (they should go through the
+            # log formatter).
+            if 'message' in data:
+                message = data['message']
+            else:
+                message = data['data']
+            error_level = self._handle_unstructured_output(message, log_output=False)
 
-        if action == "log":
-            level = getattr(log, data["level"].upper())
+            if 'level' in data:
+                level = getattr(log, data['level'].upper())
+            level = self.worst_level(error_level, level)
 
         log_data = self.formatter(data)
         if log_data is not None:
             self.log(log_data, level=level)
             self.update_levels(tbpl_level, level)
 
     def evaluate_parser(self, return_code, success_codes=None):
         success_codes = success_codes or [0]