Bug 794713 - Convert process output to Unicode; handle Unicode in logger properly; r=glandium

DONTBUILD (NPOTB)

python/mozbuild/mozbuild/base.py

@@ -9,6 +9,7 @@ import multiprocessing
 import os
 import pymake.parser
 import shlex
+import sys
 import subprocess
 import which
 
@@ -304,6 +305,10 @@ class MozbuildObject(object):
         self.log(logging.INFO, 'process', {'args': args}, ' '.join(args))
 
         def handleLine(line):
+            # Converts str to unicode on Python 2 and bytes to str on Python 3.
+            if isinstance(line, bytes):
+                line = line.decode(sys.stdout.encoding)
+
             if line_handler:
                 line_handler(line)
 

python/mozbuild/mozbuild/logger.py

@@ -97,7 +97,7 @@ class StructuredTerminalFormatter(StructuredHumanFormatter):
         elif s.startswith('TEST-UNEXPECTED'):
             result = self.terminal.red(s[0:20]) + s[21:]
 
-        return result.decode('UTF-8', 'ignore')
+        return result
 
 
 class LoggingManager(object):

python/mozbuild/mozbuild/test/test_logger.py

@@ -0,0 +1,44 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import logging
+import time
+import unittest
+
+from mozbuild.logger import StructuredHumanFormatter
+
+
+class DummyLogger(logging.Logger):
+    def __init__(self, cb):
+        logging.Logger.__init__(self, 'test')
+
+        self._cb = cb
+
+    def handle(self, record):
+        self._cb(record)
+
+
+class TestStructuredHumanFormatter(unittest.TestCase):
+    def test_non_ascii_logging(self):
+        # Ensures the formatter doesn't choke when non-ASCII characters are
+        # present in printed parameters.
+        formatter = StructuredHumanFormatter(time.time())
+
+        def on_record(record):
+            result = formatter.format(record)
+            relevant = result[5:]
+
+            self.assertEqual(relevant, 'Test: s\xe9curit\xe9')
+
+        logger = DummyLogger(on_record)
+
+        value = 's\xe9curit\xe9'
+
+        logger.log(logging.INFO, 'Test: {utf}',
+            extra={'action': 'action', 'params': {'utf': value}})
+
+
+