Bug 729098 - Ability to track and rerun failed xpcshell tests; r=gps

--HG--
extra : rebase_source : da458ded1a6574a230e127331ac4c5b24e494af2

testing/xpcshell/mach_commands.py

@@ -8,6 +8,7 @@ from __future__ import unicode_literals, print_function
 
 import mozpack.path
 import os
+import shutil
 import sys
 
 from StringIO import StringIO
@@ -46,7 +47,8 @@ class XPCShellRunner(MozbuildObject):
 
     def run_test(self, test_file, interactive=False,
                  keep_going=False, sequential=False, shuffle=False,
-                 debugger=None, debuggerArgs=None, debuggerInteractive=None):
+                 debugger=None, debuggerArgs=None, debuggerInteractive=None,
+                 rerun_failures=False):
         """Runs an individual xpcshell test."""
         # TODO Bug 794506 remove once mach integrates with virtualenv.
         build_path = os.path.join(self.topobjdir, 'build')
@@ -57,7 +59,8 @@ class XPCShellRunner(MozbuildObject):
             self.run_suite(interactive=interactive,
                            keep_going=keep_going, shuffle=shuffle, sequential=sequential,
                            debugger=debugger, debuggerArgs=debuggerArgs,
-                           debuggerInteractive=debuggerInteractive)
+                           debuggerInteractive=debuggerInteractive,
+                           rerun_failures=rerun_failures)
             return
 
         path_arg = self._wrap_path_argument(test_file)
@@ -88,6 +91,7 @@ class XPCShellRunner(MozbuildObject):
             'debugger': debugger,
             'debuggerArgs': debuggerArgs,
             'debuggerInteractive': debuggerInteractive,
+            'rerun_failures': rerun_failures
         }
 
         if os.path.isfile(path_arg.srcdir_path()):
@@ -98,7 +102,8 @@ class XPCShellRunner(MozbuildObject):
     def _run_xpcshell_harness(self, test_dirs=None, manifest=None,
                               test_path=None, shuffle=False, interactive=False,
                               keep_going=False, sequential=False,
-                              debugger=None, debuggerArgs=None, debuggerInteractive=None):
+                              debugger=None, debuggerArgs=None, debuggerInteractive=None,
+                              rerun_failures=False):
 
         # Obtain a reference to the xpcshell test runner.
         import runxpcshelltests
@@ -144,6 +149,22 @@ class XPCShellRunner(MozbuildObject):
         if test_path is not None:
             args['testPath'] = test_path
 
+        # A failure manifest is written by default. If --rerun-failures is
+        # specified and a prior failure manifest is found, the prior manifest
+        # will be run. A new failure manifest is always written over any
+        # prior failure manifest.
+        failure_manifest_path = os.path.join(self.statedir, 'xpcshell.failures.ini')
+        rerun_manifest_path = os.path.join(self.statedir, 'xpcshell.rerun.ini')
+        if os.path.exists(failure_manifest_path) and rerun_failures:
+            shutil.move(failure_manifest_path, rerun_manifest_path)
+            args['manifest'] = rerun_manifest_path
+        elif os.path.exists(failure_manifest_path):
+            os.remove(failure_manifest_path)
+        elif rerun_failures:
+            print("No failures were found to re-run.")
+            return 0
+        args['failureManifest'] = failure_manifest_path
+
         # Python through 2.7.2 has issues with unicode in some of the
         # arguments. Work around that.
         filtered_args = {}
@@ -191,6 +212,8 @@ class MachCommands(MachCommandBase):
         help='Run the tests sequentially.')
     @CommandArgument('--shuffle', '-s', action='store_true',
         help='Randomize the execution order of tests.')
+    @CommandArgument('--rerun-failures', action='store_true',
+        help='Reruns failures from last time.')
     def run_xpcshell_test(self, **params):
         # We should probably have a utility function to ensure the tree is
         # ready to run tests. Until then, we just create the state dir (in

testing/xpcshell/runxpcshelltests.py

@@ -111,6 +111,7 @@ class XPCShellTestThread(Thread):
         self.logfiles = kwargs.get('logfiles')
         self.xpcshell = kwargs.get('xpcshell')
         self.xpcsRunArgs = kwargs.get('xpcsRunArgs')
+        self.failureManifest = kwargs.get('failureManifest')
 
         self.tests_root_dir = tests_root_dir
         self.app_dir_key = app_dir_key
@@ -574,6 +575,13 @@ class XPCShellTestThread(Thread):
                   "message": message,
                   "text": stdout
                 }
+
+                if self.failureManifest:
+                    with open(self.failureManifest, 'a') as f:
+                        f.write('[%s]\n' % self.test_object['path'])
+                        for k, v in self.test_object.items():
+                            f.write('%s = %s\n' % (k, v))
+
             else:
                 now = time.time()
                 timeTaken = (now - startTime) * 1000
@@ -1062,7 +1070,8 @@ class XPCShellTests(object):
                  profileName=None, mozInfo=None, sequential=False, shuffle=False,
                  testsRootDir=None, xunitFilename=None, xunitName=None,
                  testingModulesDir=None, autolog=False, pluginsPath=None,
-                 testClass=XPCShellTestThread, **otherOptions):
+                 testClass=XPCShellTestThread, failureManifest=None,
+                 **otherOptions):
         """Run xpcshell tests.
 
         |xpcshell|, is the xpcshell executable to use to run the tests.
@@ -1226,6 +1235,7 @@ class XPCShellTests(object):
             'logfiles': self.logfiles,
             'xpcshell': self.xpcshell,
             'xpcsRunArgs': self.xpcsRunArgs,
+            'failureManifest': failureManifest
         }
 
         if self.sequential:
@@ -1440,6 +1450,9 @@ class XPCShellOptions(OptionParser):
                         help="name to record for this xUnit test suite. Many "
                              "tools expect Java class notation, e.g. "
                              "dom.basic.foo")
+        self.add_option("--failure-manifest", dest="failureManifest",
+                        action="store",
+                        help="path to file where failure manifest will be written.")
 
 def main():
     parser = XPCShellOptions()