Bug 966864 - |mach valgrind-test|: Parse Valgrind output so that different errors are distinguished for TBPL. r=gps.

--HG--
extra : rebase_source : de5bdf985c704e77ad1f9b4ab244d06141c92124

build/valgrind/mach_commands.py

@@ -5,6 +5,7 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import re
 import subprocess
 
 from mach.decorators import (
@@ -42,7 +43,6 @@ class MachCommands(MachCommandBase):
             'files.')
     def valgrind_test(self, suppressions):
         import json
-        import re
         import sys
         import tempfile
 
@@ -53,6 +53,7 @@ class MachCommands(MachCommandBase):
         from mozprofile.permissions import ServerLocations
         from mozrunner import FirefoxRunner
         from mozrunner.utils import findInPath
+        from valgrind.output_handler import OutputHandler
 
         build_dir = os.path.join(self.topsrcdir, 'build')
 
@@ -92,16 +93,6 @@ class MachCommands(MachCommandBase):
             env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
             env['XPCOM_DEBUG_BREAK'] = 'warn'
 
-            class OutputHandler(object):
-                def __init__(self):
-                    self.found_errors = False
-
-                def __call__(self, line):
-                    print(line)
-                    m = re.match(r'.*ERROR SUMMARY: [1-9]\d* errors from \d+ contexts', line)
-                    if m:
-                        self.found_errors = True
-
             outputHandler = OutputHandler()
             kp_kwargs = {'processOutputLine': [outputHandler]}
 
@@ -145,12 +136,18 @@ class MachCommands(MachCommandBase):
                 exitcode = runner.wait()
 
             finally:
-                if not outputHandler.found_errors:
+                errs = outputHandler.error_count
+                supps = outputHandler.suppression_count
+                if errs != supps:
+                    status = 1  # turns the TBPL job orange
+                    print('TEST-UNEXPECTED-FAILURE | valgrind-test | error parsing:', errs, "errors seen, but", supps, "generated suppressions seen")
+
+                elif errs == 0:
                     status = 0
                     print('TEST-PASS | valgrind-test | valgrind found no errors')
                 else:
                     status = 1  # turns the TBPL job orange
-                    print('TEST-UNEXPECTED-FAIL | valgrind-test | valgrind found errors')
+                    # We've already printed details of the errors.
 
                 if exitcode != 0:
                     status = 2  # turns the TBPL job red

build/valgrind/output_handler.py

@@ -0,0 +1,105 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import re
+
+class OutputHandler(object):
+    '''
+    A class for handling Valgrind output.
+
+    Valgrind errors look like this:
+
+    ==60741== 40 (24 direct, 16 indirect) bytes in 1 blocks are definitely lost in loss record 2,746 of 5,235
+    ==60741==    at 0x4C26B43: calloc (vg_replace_malloc.c:593)
+    ==60741==    by 0x63AEF65: PR_Calloc (prmem.c:443)
+    ==60741==    by 0x69F236E: PORT_ZAlloc_Util (secport.c:117)
+    ==60741==    by 0x69F1336: SECITEM_AllocItem_Util (secitem.c:28)
+    ==60741==    by 0xA04280B: ffi_call_unix64 (in /builds/slave/m-in-l64-valgrind-000000000000/objdir/toolkit/library/libxul.so)
+    ==60741==    by 0xA042443: ffi_call (ffi64.c:485)
+
+    For each such error, this class extracts most or all of the first (error
+    kind) line, plus the function name in each of the first few stack entries.
+    With this data it constructs and prints a TEST-UNEXPECTED-FAIL message that
+    TBPL will highlight.
+
+    It buffers these lines from which text is extracted so that the
+    TEST-UNEXPECTED-FAIL message can be printed before the full error.
+
+    Parsing the Valgrind output isn't ideal, and it may break in the future if
+    Valgrind changes the format of the messages, or introduces new error kinds.
+    To protect against this, we also count how many lines containing
+    "<insert_a_suppression_name_here>" are seen. Thanks to the use of
+    --gen-suppressions=yes, exactly one of these lines is present per error. If
+    the count of these lines doesn't match the error count found during
+    parsing, then the parsing has missed one or more errors and we can fail
+    appropriately.
+    '''
+
+    def __init__(self):
+        # The regexps in this list match all of Valgrind's errors. Note that
+        # Valgrind is English-only, so we don't have to worry about
+        # localization.
+        self.re_error = \
+            r'==\d+== (' + \
+            r'(Use of uninitialised value of size \d+)|' + \
+            r'(Conditional jump or move depends on uninitialised value\(s\))|' + \
+            r'(Syscall param .* contains uninitialised byte\(s\))|' + \
+            r'(Syscall param .* points to (unaddressable|uninitialised) byte\(s\))|' + \
+            r'((Unaddressable|Uninitialised) byte\(s\) found during client check request)|' + \
+            r'(Invalid free\(\) / delete / delete\[\] / realloc\(\))|' + \
+            r'(Mismatched free\(\) / delete / delete \[\])|' + \
+            r'(Invalid (read|write) of size \d+)|' + \
+            r'(Jump to the invalid address stated on the next line)|' + \
+            r'(Source and destination overlap in .*)|' + \
+            r'(.* bytes in .* blocks are .* lost)' + \
+            r')'
+        # Match identifer chars, plus ':' for namespaces, and '\?' in order to
+        # match "???" which Valgrind sometimes produces.
+        self.re_stack_entry = r'^==\d+==.*0x[A-Z0-9]+: ([A-Za-z0_9_:\?]+)'
+        self.re_suppression = r' *<insert_a_suppression_name_here>'
+        self.error_count = 0
+        self.suppression_count = 0
+        self.number_of_stack_entries_to_get = 0
+        self.curr_failure_msg = None
+        self.buffered_lines = None
+
+    def __call__(self, line):
+        if self.number_of_stack_entries_to_get == 0:
+            # Look for the start of a Valgrind error.
+            m = re.search(self.re_error, line)
+            if m:
+                self.error_count += 1
+                self.number_of_stack_entries_to_get = 4
+                self.curr_failure_msg = 'TEST-UNEXPECTED-FAIL | valgrind-test | ' + m.group(1) + " at "
+                self.buffered_lines = [line]
+            else:
+                print(line)
+
+        else:
+            # We've recently found a Valgrind error, and are now extracting
+            # details from the first few stack entries.
+            self.buffered_lines.append(line)
+            m = re.match(self.re_stack_entry, line)
+            if m:
+                self.curr_failure_msg += m.group(1)
+            else:
+                self.curr_failure_msg += '?!?'
+
+            self.number_of_stack_entries_to_get -= 1
+            if self.number_of_stack_entries_to_get != 0:
+                self.curr_failure_msg += ' / '
+            else:
+                # We've finished getting the first few stack entries. Print the
+                # failure message and the buffered lines, and then reset state.
+                print('\n' + self.curr_failure_msg + '\n')
+                for b in self.buffered_lines:
+                    print(b)
+                self.curr_failure_msg = None
+                self.buffered_lines = None
+
+        if re.match(self.re_suppression, line):
+            self.suppression_count += 1
+