From c9ac45fd01f6484aa939266dfbe686f2865d9e87 Mon Sep 17 00:00:00 2001
From: Gregory Szorc <gps@mozilla.com>
Date: Fri, 28 Sep 2012 10:29:32 -0700
Subject: [PATCH] Bug 794713 - Convert process output to Unicode; handle
 Unicode in logger properly; r=glandium

DONTBUILD (NPOTB)
---
 python/mozbuild/mozbuild/base.py             |  5 +++
 python/mozbuild/mozbuild/logger.py           |  2 +-
 python/mozbuild/mozbuild/test/test_logger.py | 44 ++++++++++++++++++++
 3 files changed, 50 insertions(+), 1 deletion(-)
 create mode 100644 python/mozbuild/mozbuild/test/test_logger.py

diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py
index 268bd830888..58a6d4d2a8a 100644
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -9,6 +9,7 @@ import multiprocessing
 import os
 import pymake.parser
 import shlex
+import sys
 import subprocess
 import which
 
@@ -304,6 +305,10 @@ class MozbuildObject(object):
         self.log(logging.INFO, 'process', {'args': args}, ' '.join(args))
 
         def handleLine(line):
+            # Converts str to unicode on Python 2 and bytes to str on Python 3.
+            if isinstance(line, bytes):
+                line = line.decode(sys.stdout.encoding)
+
             if line_handler:
                 line_handler(line)
 
diff --git a/python/mozbuild/mozbuild/logger.py b/python/mozbuild/mozbuild/logger.py
index 6e91a522c29..2bebf62bbf1 100644
--- a/python/mozbuild/mozbuild/logger.py
+++ b/python/mozbuild/mozbuild/logger.py
@@ -97,7 +97,7 @@ class StructuredTerminalFormatter(StructuredHumanFormatter):
         elif s.startswith('TEST-UNEXPECTED'):
             result = self.terminal.red(s[0:20]) + s[21:]
 
-        return result.decode('UTF-8', 'ignore')
+        return result
 
 
 class LoggingManager(object):
diff --git a/python/mozbuild/mozbuild/test/test_logger.py b/python/mozbuild/mozbuild/test/test_logger.py
new file mode 100644
index 00000000000..7b5a10c6c8e
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/test_logger.py
@@ -0,0 +1,44 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import logging
+import time
+import unittest
+
+from mozbuild.logger import StructuredHumanFormatter
+
+
+class DummyLogger(logging.Logger):
+    def __init__(self, cb):
+        logging.Logger.__init__(self, 'test')
+
+        self._cb = cb
+
+    def handle(self, record):
+        self._cb(record)
+
+
+class TestStructuredHumanFormatter(unittest.TestCase):
+    def test_non_ascii_logging(self):
+        # Ensures the formatter doesn't choke when non-ASCII characters are
+        # present in printed parameters.
+        formatter = StructuredHumanFormatter(time.time())
+
+        def on_record(record):
+            result = formatter.format(record)
+            relevant = result[5:]
+
+            self.assertEqual(relevant, 'Test: s\xe9curit\xe9')
+
+        logger = DummyLogger(on_record)
+
+        value = 's\xe9curit\xe9'
+
+        logger.log(logging.INFO, 'Test: {utf}',
+            extra={'action': 'action', 'params': {'utf': value}})
+
+
+