Bug 886741 - Fix support for expectedFailure and skip, r=jgriffin

testing/marionette/client/marionette/__init__.py

@@ -4,7 +4,7 @@
 
 from gestures import *
 from marionette import Marionette, HTMLElement, Actions, MultiActions
-from marionette_test import MarionetteTestCase, CommonTestCase
+from marionette_test import MarionetteTestCase, CommonTestCase, expectedFailure, skip, SkipTest
 from emulator import Emulator
 from runtests import MarionetteTestResult
 from runtests import MarionetteTestRunner

testing/marionette/client/marionette/marionette_test.py

@@ -5,15 +5,69 @@
 import imp
 import os
 import re
+import functools
 import sys
 import time
 import types
 import unittest
 import weakref
+import warnings
 
 from errors import *
 from marionette import HTMLElement, Marionette
 
+class SkipTest(Exception):
+    """
+    Raise this exception in a test to skip it.
+
+    Usually you can use TestResult.skip() or one of the skipping decorators
+    instead of raising this directly.
+    """
+    pass
+
+class _ExpectedFailure(Exception):
+    """
+    Raise this when a test is expected to fail.
+
+    This is an implementation detail.
+    """
+
+    def __init__(self, exc_info):
+        super(_ExpectedFailure, self).__init__()
+        self.exc_info = exc_info
+
+class _UnexpectedSuccess(Exception):
+    """
+    The test was supposed to fail, but it didn't!
+    """
+    pass
+
+def skip(reason):
+    """
+    Unconditionally skip a test.
+    """
+    def decorator(test_item):
+        if not isinstance(test_item, (type, types.ClassType)):
+            @functools.wraps(test_item)
+            def skip_wrapper(*args, **kwargs):
+                raise SkipTest(reason)
+            test_item = skip_wrapper
+
+        test_item.__unittest_skip__ = True
+        test_item.__unittest_skip_why__ = reason
+        return test_item
+    return decorator
+
+def expectedFailure(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            func(*args, **kwargs)
+        except Exception:
+            raise _ExpectedFailure(sys.exc_info())
+        raise _UnexpectedSuccess
+    return wrapper
+
 def skip_if_b2g(target):
     def wrapper(self, *args, **kwargs):
         if not hasattr(self.marionette, 'b2g') or not self.marionette.b2g:
@@ -25,12 +79,22 @@ def skip_if_b2g(target):
 class CommonTestCase(unittest.TestCase):
 
     match_re = None
+    failureException = AssertionError
 
     def __init__(self, methodName):
         unittest.TestCase.__init__(self, methodName)
         self.loglines = None
         self.duration = 0
 
+    def _addSkip(self, result, reason):
+        addSkip = getattr(result, 'addSkip', None)
+        if addSkip is not None:
+            addSkip(self, reason)
+        else:
+            warnings.warn("TestResult has no addSkip method, skips not reported",
+                          RuntimeWarning, 2)
+            result.addSuccess(self)
+
     def run(self, result=None):
         orig_result = result
         if result is None:
@@ -39,14 +103,25 @@ class CommonTestCase(unittest.TestCase):
             if startTestRun is not None:
                 startTestRun()
 
-        self._resultForDoCleanups = result
         result.startTest(self)
 
         testMethod = getattr(self, self._testMethodName)
+        if (getattr(self.__class__, "__unittest_skip__", False) or
+            getattr(testMethod, "__unittest_skip__", False)):
+            # If the class or method was skipped.
+            try:
+                skip_why = (getattr(self.__class__, '__unittest_skip_why__', '')
+                            or getattr(testMethod, '__unittest_skip_why__', ''))
+                self._addSkip(result, skip_why)
+            finally:
+                result.stopTest(self)
+            return
         try:
             success = False
             try:
                 self.setUp()
+            except SkipTest as e:
+                self._addSkip(result, str(e))
             except KeyboardInterrupt:
                 raise
             except:
@@ -54,10 +129,30 @@ class CommonTestCase(unittest.TestCase):
             else:
                 try:
                     testMethod()
+                except self.failureException:
+                    result.addFailure(self, sys.exc_info())
                 except KeyboardInterrupt:
                     raise
-                except AssertionError:
+                except self.failureException:
                     result.addFailure(self, sys.exc_info())
+                except _ExpectedFailure as e:
+                    addExpectedFailure = getattr(result, 'addExpectedFailure', None)
+                    if addExpectedFailure is not None:
+                        addExpectedFailure(self, e.exc_info)
+                    else:
+                        warnings.warn("TestResult has no addExpectedFailure method, reporting as passes",
+                                      RuntimeWarning)
+                        result.addSuccess(self)
+                except _UnexpectedSuccess:
+                    addUnexpectedSuccess = getattr(result, 'addUnexpectedSuccess', None)
+                    if addUnexpectedSuccess is not None:
+                        addUnexpectedSuccess(self)
+                    else:
+                        warnings.warn("TestResult has no addUnexpectedSuccess method, reporting as failures",
+                                      RuntimeWarning)
+                        result.addFailure(self, sys.exc_info())
+                except SkipTest as e:
+                    self._addSkip(result, str(e))
                 except:
                     result.addError(self, sys.exc_info())
                 else:

testing/marionette/client/marionette/runtests.py

@@ -16,6 +16,7 @@ import traceback
 import platform
 import moznetwork
 import xml.dom.minidom as dom
+from functools import wraps
 
 from manifestparser import TestManifest
 from mozhttpd import MozHttpd
@@ -32,6 +33,9 @@ class MarionetteTestResult(unittest._TextTestResult):
         del kwargs['marionette']
         super(MarionetteTestResult, self).__init__(*args, **kwargs)
         self.passed = 0
+        self.skipped = []
+        self.expectedFailures = []
+        self.unexpectedSuccesses = []
         self.tests_passed = []
 
     def addSuccess(self, test):
@@ -39,6 +43,33 @@ class MarionetteTestResult(unittest._TextTestResult):
         self.passed += 1
         self.tests_passed.append(test)
 
+    def addExpectedFailure(self, test, err):
+        """Called when an expected failure/error occured."""
+        self.expectedFailures.append(
+            (test, self._exc_info_to_string(err, test)))
+        if self.showAll:
+            self.stream.writeln("expected failure")
+        elif self.dots:
+            self.stream.write("x")
+            self.stream.flush()
+
+    def addUnexpectedSuccess(self, test):
+        """Called when a test was expected to fail, but succeed."""
+        self.unexpectedSuccesses.append(test)
+        if self.showAll:
+            self.stream.writeln("unexpected success")
+        elif self.dots:
+            self.stream.write("u")
+            self.stream.flush()
+
+    def addSkip(self, test, reason):
+        self.skipped.append((test, reason))
+        if self.showAll:
+            self.stream.writeln("skipped {0!r}".format(reason))
+        elif self.dots:
+            self.stream.write("s")
+            self.stream.flush()
+
     def getInfo(self, test):
         return test.test_name
 
@@ -491,14 +522,16 @@ class MarionetteTestRunner(object):
 
             self.failed += len(results.failures) + len(results.errors)
             if hasattr(results, 'skipped'):
-                self.todo += len(results.skipped) + len(results.expectedFailures)
+                self.todo += len(results.skipped)
             self.passed += results.passed
             for failure in results.failures + results.errors:
                 self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL'))
-            if hasattr(results, 'unexpectedSuccess'):
+            if hasattr(results, 'unexpectedSuccesses'):
                 self.failed += len(results.unexpectedSuccesses)
                 for failure in results.unexpectedSuccesses:
-                    self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-PASS'))
+                    self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS'))
+            if hasattr(results, 'expectedFailures'):
+                self.passed += len(results.expectedFailures)
 
     def register_handlers(self):
         self.test_handlers.extend([MarionetteTestCase, MarionetteJSTestCase])

testing/marionette/client/marionette/tests/unit/test_report.py

@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+from marionette_test import MarionetteTestCase, expectedFailure, skip
+
+
+class TestReport(MarionetteTestCase):
+
+    def test_pass(self):
+        assert True
+
+    def test_fail(self):
+        assert False
+
+    @skip('Skip Message')
+    def test_skip(self):
+        assert False
+
+    @expectedFailure
+    def test_expected_fail(self):
+        assert False
+
+    @expectedFailure
+    def test_unexpected_pass(self):
+        assert True
+
+    def test_error(self):
+        raise Exception()