Bug 966864 - |mach valgrind-test|: Parse Valgrind output so that different errors are distinguished for TBPL. r=gps.
authorNicholas Nethercote <nnethercote@mozilla.com>
Wed, 05 Feb 2014 16:17:09 -0800
changeset 167784 ce819018434bb72961b9c764e9de8b1c9a29930b
parent 167783 b667bd72d90978516a1b686a280208639a50f799
child 167794 ecf20a2484b6aaeb50c9a7f08272646923a9dff1
child 167795 1bab733c5a83508638332db9b86473649eb92fa9
push id26184
push userphilringnalda@gmail.com
push dateMon, 10 Feb 2014 02:32:38 +0000
treeherdermozilla-central@ce819018434b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs966864
milestone30.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 966864 - |mach valgrind-test|: Parse Valgrind output so that different errors are distinguished for TBPL. r=gps.
build/valgrind/__init__.py
build/valgrind/mach_commands.py
build/valgrind/output_handler.py
new file mode 100644
--- a/build/valgrind/mach_commands.py
+++ b/build/valgrind/mach_commands.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function, unicode_literals
 
 import os
+import re
 import subprocess
 
 from mach.decorators import (
     Command,
     CommandArgument,
     CommandProvider,
 )
 from mozbuild.base import (
@@ -37,27 +38,27 @@ class MachCommands(MachCommandBase):
         description='Run the Valgrind test job.')
     @CommandArgument('--suppressions', default=[], action='append',
         metavar='FILENAME',
         help='Specify a suppression file for Valgrind to use. Use '
             '--suppression multiple times to specify multiple suppression '
             'files.')
     def valgrind_test(self, suppressions):
         import json
-        import re
         import sys
         import tempfile
 
         from mozbuild.base import MozbuildObject
         from mozfile import TemporaryDirectory
         from mozhttpd import MozHttpd
         from mozprofile import FirefoxProfile, Preferences
         from mozprofile.permissions import ServerLocations
         from mozrunner import FirefoxRunner
         from mozrunner.utils import findInPath
+        from valgrind.output_handler import OutputHandler
 
         build_dir = os.path.join(self.topsrcdir, 'build')
 
         # XXX: currently we just use the PGO inputs for Valgrind runs.  This may
         # change in the future.
         httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
         httpd.start(block=False)
 
@@ -87,26 +88,16 @@ class MachCommands(MachCommandBase):
             firefox_args = [httpd.get_url()]
 
             env = os.environ.copy()
             env['G_SLICE'] = 'always-malloc'
             env['XPCOM_CC_RUN_DURING_SHUTDOWN'] = '1'
             env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
             env['XPCOM_DEBUG_BREAK'] = 'warn'
 
-            class OutputHandler(object):
-                def __init__(self):
-                    self.found_errors = False
-
-                def __call__(self, line):
-                    print(line)
-                    m = re.match(r'.*ERROR SUMMARY: [1-9]\d* errors from \d+ contexts', line)
-                    if m:
-                        self.found_errors = True
-
             outputHandler = OutputHandler()
             kp_kwargs = {'processOutputLine': [outputHandler]}
 
             valgrind = 'valgrind'
             if not os.path.exists(valgrind):
                 valgrind = findInPath(valgrind)
 
             valgrind_args = [
@@ -140,22 +131,28 @@ class MachCommands(MachCommandBase):
                                        binary=self.get_binary_path(),
                                        cmdargs=firefox_args,
                                        env=env,
                                        kp_kwargs=kp_kwargs)
                 runner.start(debug_args=valgrind_args)
                 exitcode = runner.wait()
 
             finally:
-                if not outputHandler.found_errors:
+                errs = outputHandler.error_count
+                supps = outputHandler.suppression_count
+                if errs != supps:
+                    status = 1  # turns the TBPL job orange
+                    print('TEST-UNEXPECTED-FAILURE | valgrind-test | error parsing:', errs, "errors seen, but", supps, "generated suppressions seen")
+
+                elif errs == 0:
                     status = 0
                     print('TEST-PASS | valgrind-test | valgrind found no errors')
                 else:
                     status = 1  # turns the TBPL job orange
-                    print('TEST-UNEXPECTED-FAIL | valgrind-test | valgrind found errors')
+                    # We've already printed details of the errors.
 
                 if exitcode != 0:
                     status = 2  # turns the TBPL job red
                     print('TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind')
 
                 httpd.stop()
 
             return status
new file mode 100644
--- /dev/null
+++ b/build/valgrind/output_handler.py
@@ -0,0 +1,105 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import re
+
+class OutputHandler(object):
+    '''
+    A class for handling Valgrind output.
+
+    Valgrind errors look like this:
+
+    ==60741== 40 (24 direct, 16 indirect) bytes in 1 blocks are definitely lost in loss record 2,746 of 5,235
+    ==60741==    at 0x4C26B43: calloc (vg_replace_malloc.c:593)
+    ==60741==    by 0x63AEF65: PR_Calloc (prmem.c:443)
+    ==60741==    by 0x69F236E: PORT_ZAlloc_Util (secport.c:117)
+    ==60741==    by 0x69F1336: SECITEM_AllocItem_Util (secitem.c:28)
+    ==60741==    by 0xA04280B: ffi_call_unix64 (in /builds/slave/m-in-l64-valgrind-000000000000/objdir/toolkit/library/libxul.so)
+    ==60741==    by 0xA042443: ffi_call (ffi64.c:485)
+
+    For each such error, this class extracts most or all of the first (error
+    kind) line, plus the function name in each of the first few stack entries.
+    With this data it constructs and prints a TEST-UNEXPECTED-FAIL message that
+    TBPL will highlight.
+
+    It buffers these lines from which text is extracted so that the
+    TEST-UNEXPECTED-FAIL message can be printed before the full error.
+
+    Parsing the Valgrind output isn't ideal, and it may break in the future if
+    Valgrind changes the format of the messages, or introduces new error kinds.
+    To protect against this, we also count how many lines containing
+    "<insert_a_suppression_name_here>" are seen. Thanks to the use of
+    --gen-suppressions=yes, exactly one of these lines is present per error. If
+    the count of these lines doesn't match the error count found during
+    parsing, then the parsing has missed one or more errors and we can fail
+    appropriately.
+    '''
+
+    def __init__(self):
+        # The regexps in this list match all of Valgrind's errors. Note that
+        # Valgrind is English-only, so we don't have to worry about
+        # localization.
+        self.re_error = \
+            r'==\d+== (' + \
+            r'(Use of uninitialised value of size \d+)|' + \
+            r'(Conditional jump or move depends on uninitialised value\(s\))|' + \
+            r'(Syscall param .* contains uninitialised byte\(s\))|' + \
+            r'(Syscall param .* points to (unaddressable|uninitialised) byte\(s\))|' + \
+            r'((Unaddressable|Uninitialised) byte\(s\) found during client check request)|' + \
+            r'(Invalid free\(\) / delete / delete\[\] / realloc\(\))|' + \
+            r'(Mismatched free\(\) / delete / delete \[\])|' + \
+            r'(Invalid (read|write) of size \d+)|' + \
+            r'(Jump to the invalid address stated on the next line)|' + \
+            r'(Source and destination overlap in .*)|' + \
+            r'(.* bytes in .* blocks are .* lost)' + \
+            r')'
+        # Match identifer chars, plus ':' for namespaces, and '\?' in order to
+        # match "???" which Valgrind sometimes produces.
+        self.re_stack_entry = r'^==\d+==.*0x[A-Z0-9]+: ([A-Za-z0_9_:\?]+)'
+        self.re_suppression = r' *<insert_a_suppression_name_here>'
+        self.error_count = 0
+        self.suppression_count = 0
+        self.number_of_stack_entries_to_get = 0
+        self.curr_failure_msg = None
+        self.buffered_lines = None
+
+    def __call__(self, line):
+        if self.number_of_stack_entries_to_get == 0:
+            # Look for the start of a Valgrind error.
+            m = re.search(self.re_error, line)
+            if m:
+                self.error_count += 1
+                self.number_of_stack_entries_to_get = 4
+                self.curr_failure_msg = 'TEST-UNEXPECTED-FAIL | valgrind-test | ' + m.group(1) + " at "
+                self.buffered_lines = [line]
+            else:
+                print(line)
+
+        else:
+            # We've recently found a Valgrind error, and are now extracting
+            # details from the first few stack entries.
+            self.buffered_lines.append(line)
+            m = re.match(self.re_stack_entry, line)
+            if m:
+                self.curr_failure_msg += m.group(1)
+            else:
+                self.curr_failure_msg += '?!?'
+
+            self.number_of_stack_entries_to_get -= 1
+            if self.number_of_stack_entries_to_get != 0:
+                self.curr_failure_msg += ' / '
+            else:
+                # We've finished getting the first few stack entries. Print the
+                # failure message and the buffered lines, and then reset state.
+                print('\n' + self.curr_failure_msg + '\n')
+                for b in self.buffered_lines:
+                    print(b)
+                self.curr_failure_msg = None
+                self.buffered_lines = None
+
+        if re.match(self.re_suppression, line):
+            self.suppression_count += 1
+