From 7653ba13dbb28d0b396007e0447370a05567ed11 Mon Sep 17 00:00:00 2001
From: Maja Frydrychowicz <mjzffr@gmail.com>
Date: Tue, 10 Nov 2015 13:09:13 -0500
Subject: [PATCH] Bug 1223429 - Return exit code 10 when Marionette harness has
 failing tests; r=automatedtester

---
 .../marionette/client/marionette/runtests.py  | 24 +++++++++++++++----
 testing/marionette/mach_commands.py           |  6 ++---
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/testing/marionette/client/marionette/runtests.py b/testing/marionette/client/marionette/runtests.py
index 4fbe6b7c8ee..db959a75749 100644
--- a/testing/marionette/client/marionette/runtests.py
+++ b/testing/marionette/client/marionette/runtests.py
@@ -69,14 +69,30 @@ class MarionetteHarness(object):
             tests = args_dict.pop('tests')
             runner = self._runner_class(**args_dict)
             runner.run_tests(tests)
-            return runner
+            return runner.failed
         except Exception:
             self.args.logger.error('Failure during test execution.',
                                    exc_info=True)
-            sys.exit(1)
+            raise
 
-def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments):
-    MarionetteHarness(runner_class, parser_class).run()
+
+def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments,
+        harness_class=MarionetteHarness):
+    """
+    Call the harness to parse args and run tests.
+
+    The following exit codes are expected:
+    - Test failures: 10
+    - Harness/other failures: 1
+    - Success: 0
+    """
+    try:
+        failed = harness_class(runner_class, parser_class).run()
+        if failed > 0:
+            sys.exit(10)
+    except Exception:
+        sys.exit(1)
+    sys.exit(0)
 
 if __name__ == "__main__":
     cli()
diff --git a/testing/marionette/mach_commands.py b/testing/marionette/mach_commands.py
index 955e61b9245..e9a1b64a6e7 100644
--- a/testing/marionette/mach_commands.py
+++ b/testing/marionette/mach_commands.py
@@ -67,10 +67,8 @@ def run_marionette(tests, b2g_path=None, emulator=None, testtype=None,
     args.logger = commandline.setup_logging("Marionette Unit Tests",
                                             args,
                                             {"mach": sys.stdout})
-
-    results = MarionetteHarness(MarionetteTestRunner, args=args).run()
-
-    if results.failed > 0:
+    failed = MarionetteHarness(MarionetteTestRunner, args=args).run()
+    if failed > 0:
         return 1
     else:
         return 0