Bug 909888 - Add mach commands for b2g reftests and crashtests, r=jgriffin, DONTBUILD, a=NPOTB

layout/tools/reftest/mach_commands.py

@@ -7,9 +7,12 @@ from __future__ import unicode_literals
 import mozpack.path
 import os
 import re
+import sys
+import which
 
 from mozbuild.base import (
     MachCommandBase,
+    MachCommandConditions as conditions,
     MozbuildObject,
 )
 
@@ -22,6 +25,8 @@ from mach.decorators import (
 
 DEBUGGER_HELP = 'Debugger binary to run test in. Program name or path.'
 
+ADB_NOT_FOUND = """The %s command requires the adb binary to be on your path.
+This can be found in '%s/out/host/<platform>/bin'."""
 
 class ReftestRunner(MozbuildObject):
     """Easily run reftests.
@@ -29,6 +34,16 @@ class ReftestRunner(MozbuildObject):
     This currently contains just the basics for running reftests. We may want
     to hook up result parsing, etc.
     """
+    def __init__(self, *args, **kwargs):
+        MozbuildObject.__init__(self, *args, **kwargs)
+
+        # TODO Bug 794506 remove once mach integrates with virtualenv.
+        build_path = os.path.join(self.topobjdir, 'build')
+        if build_path not in sys.path:
+            sys.path.append(build_path)
+
+        self.tests_dir = os.path.join(self.topobjdir, '_tests')
+        self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
 
     def _manifest_file(self, suite):
         """Returns the manifest file used for a given test suite."""
@@ -57,7 +72,72 @@ class ReftestRunner(MozbuildObject):
     def _make_shell_string(self, s):
         return "'%s'" % re.sub("'", r"'\''", s)
 
-    def run_reftest_test(self, test_file=None, filter=None, suite=None,
+    def run_b2g_test(self, b2g_home, xre_path, test_file=None, suite=None, **kwargs):
+        """Runs a b2g reftest.
+
+        test_file is a path to a test file. It can be a relative path from the
+        top source directory, an absolute filename, or a directory containing
+        test files.
+
+        suite is the type of reftest to run. It can be one of ('reftest',
+        'crashtest').
+        """
+        if suite not in ('reftest', 'crashtest'):
+            raise Exception('None or unrecognized reftest suite type.')
+
+        try:
+            which.which('adb')
+        except which.WhichError:
+            # TODO Find adb automatically if it isn't on the path
+            raise Exception(ADB_NOT_FOUND % ('%s-remote' % suite, b2g_home))
+
+        # Find the manifest file
+        if not test_file:
+            if suite == 'reftest':
+                test_file = mozpack.path.join('layout', 'reftests')
+            elif suite == 'crashtest':
+                test_file = mozpack.path.join('testing', 'crashtest')
+
+        if not os.path.exists(os.path.join(self.topsrcdir, test_file)):
+            test_file = mozpack.path.relpath(os.path.abspath(test_file),
+                                             self.topsrcdir)
+
+        manifest = self._find_manifest(suite, test_file)
+        if not os.path.exists(mozpack.path.join(self.topsrcdir, manifest)):
+            raise Exception('No manifest file was found at %s.' % manifest)
+
+        # Need to chdir to reftest_dir otherwise imports fail below.
+        os.chdir(self.reftest_dir)
+
+        import imp
+        path = os.path.join(self.reftest_dir, 'runreftestb2g.py')
+        with open(path, 'r') as fh:
+            imp.load_module('reftest', fh, path, ('.py', 'r', imp.PY_SOURCE))
+        import reftest
+
+        # Set up the reftest options.
+        parser = reftest.B2GOptions()
+        options, args = parser.parse_args([])
+
+        options.b2gPath = b2g_home
+        options.logcat_dir = self.reftest_dir
+        options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
+        options.ignoreWindowSize = True
+        options.xrePath = xre_path
+
+        for k, v in kwargs.iteritems():
+            setattr(options, k, v)
+
+        # Tests need to be served from a subdirectory of the server. Symlink
+        # topsrcdir here to get around this.
+        tests = os.path.join(self.reftest_dir, 'tests')
+        if not os.path.isdir(tests):
+            os.symlink(self.topsrcdir, tests)
+        args.insert(0, os.path.join('tests', manifest))
+
+        return reftest.run_remote_reftests(parser, options, args)
+
+    def run_desktop_test(self, test_file=None, filter=None, suite=None,
             debugger=None):
         """Runs a reftest.
 
@@ -124,6 +204,47 @@ def ReftestCommand(func):
 
     return func
 
+def B2GCommand(func):
+    """Decorator that adds shared command arguments to b2g mochitest commands."""
+
+    busybox = CommandArgument('--busybox', default=None,
+        help='Path to busybox binary to install on device')
+    func = busybox(func)
+
+    logcatdir = CommandArgument('--logcat-dir', default=None,
+        help='directory to store logcat dump files')
+    func = logcatdir(func)
+
+    geckopath = CommandArgument('--gecko-path', default=None,
+        help='the path to a gecko distribution that should \
+              be installed on the emulator prior to test')
+    func = geckopath(func)
+
+    sdcard = CommandArgument('--sdcard', default="10MB",
+        help='Define size of sdcard: 1MB, 50MB...etc')
+    func = sdcard(func)
+
+    emulator_res = CommandArgument('--emulator-res', default='800x1000',
+        help='Emulator resolution of the format \'<width>x<height>\'')
+    func = emulator_res(func)
+
+    emulator = CommandArgument('--emulator', default='arm',
+        help='Architecture of emulator to use: x86 or arm')
+    func = emulator(func)
+
+    marionette = CommandArgument('--marionette', default=None,
+        help='host:port to use when connecting to Marionette')
+    func = marionette(func)
+
+    path = CommandArgument('test_file', default=None, nargs='?',
+        metavar='TEST',
+        help='Test to run. Can be specified as a single file, a ' \
+            'directory, or omitted. If omitted, the entire test suite is ' \
+            'executed.')
+    func = path(func)
+
+    return func
+
 
 @CommandProvider
 class MachCommands(MachCommandBase):
@@ -150,10 +271,41 @@ class MachCommands(MachCommandBase):
     def run_crashtest_ipc(self, test_file, **kwargs):
         return self._run_reftest(test_file, suite='crashtest-ipc', **kwargs)
 
-    def _run_reftest(self, test_file=None, filter=None, suite=None,
-            debugger=None):
+    def _run_reftest(self, test_file=None, suite=None, **kwargs):
         reftest = self._spawn(ReftestRunner)
-        return reftest.run_reftest_test(test_file, filter=filter, suite=suite,
-            debugger=debugger)
+        return reftest.run_desktop_test(test_file, suite=suite, **kwargs)
 
 
+# TODO For now b2g commands will only work with the emulator,
+# they should be modified to work with all devices.
+def is_emulator(cls):
+    """Emulator needs to be configured."""
+    return cls.device_name in ('emulator', 'emulator-jb')
+
+
+@CommandProvider
+class B2GCommands(MachCommandBase):
+    def __init__(self, context):
+        MachCommandBase.__init__(self, context)
+
+        for attr in ('b2g_home', 'xre_path', 'device_name'):
+            setattr(self, attr, getattr(context, attr, None))
+
+    @Command('reftest-remote', category='testing',
+        description='Run a remote reftest.',
+        conditions=[conditions.is_b2g, is_emulator])
+    @B2GCommand
+    def run_reftest_remote(self, test_file, **kwargs):
+        return self._run_reftest(test_file, suite='reftest', **kwargs)
+
+    @Command('crashtest-remote', category='testing',
+        description='Run a remote crashtest.',
+        conditions=[conditions.is_b2g, is_emulator])
+    @B2GCommand
+    def run_crashtest_remote(self, test_file, **kwargs):
+        return self._run_reftest(test_file, suite='crashtest', **kwargs)
+
+    def _run_reftest(self, test_file=None, suite=None, **kwargs):
+        reftest = self._spawn(ReftestRunner)
+        return reftest.run_b2g_test(self.b2g_home, self.xre_path,
+            test_file, suite=suite, **kwargs)

layout/tools/reftest/runreftestb2g.py

@@ -24,8 +24,11 @@ from marionette import Marionette
 
 class B2GOptions(ReftestOptions):
 
-    def __init__(self, automation, **kwargs):
+    def __init__(self, automation=None, **kwargs):
         defaults = {}
+        if not automation:
+            automation = B2GRemoteAutomation(None, "fennec", context_chrome=True)
+
         ReftestOptions.__init__(self, automation)
 
         self.add_option("--b2gpath", action="store",
@@ -470,11 +473,8 @@ class B2GReftest(RefTest):
     def getManifestPath(self, path):
         return path
 
-
-def main(args=sys.argv[1:]):
+def run_remote_reftests(parser, options, args):
     auto = B2GRemoteAutomation(None, "fennec", context_chrome=True)
-    parser = B2GOptions(auto)
-    options, args = parser.parse_args(args)
 
     # create our Marionette instance
     kwargs = {}
@@ -508,6 +508,7 @@ def main(args=sys.argv[1:]):
     if options.deviceIP:
         kwargs.update({'host': options.deviceIP,
                        'port': options.devicePort})
+
     dm = DeviceManagerADB(**kwargs)
     auto.setDeviceManager(dm)
 
@@ -581,6 +582,12 @@ def main(args=sys.argv[1:]):
     reftest.stopWebServer(options)
     return retVal
 
+def main(args=sys.argv[1:]):
+    parser = B2GOptions()
+    options, args = parser.parse_args(args)
+    return run_remote_reftests(auto, parser, options, args)
+
+
 if __name__ == "__main__":
     sys.exit(main())
 

testing/mochitest/mach_commands.py

@@ -9,6 +9,7 @@ import mozpack.path
 import os
 import platform
 import sys
+import which
 
 from mozbuild.base import (
     MachCommandBase,
@@ -24,6 +25,8 @@ from mach.decorators import (
 
 from mach.logging import StructuredHumanFormatter
 
+ADB_NOT_FOUND = """The %s command requires the adb binary to be on your path.
+This can be found in '%s/out/host/<platform>/bin'."""
 
 class UnexpectedFilter(logging.Filter):
     def filter(self, record):
@@ -56,6 +59,12 @@ class MochitestRunner(MozbuildObject):
         top source directory, an absolute filename, or a directory containing
         test files.
         """
+        try:
+            which.which('adb')
+        except which.WhichError:
+            # TODO Find adb automatically if it isn't on the path
+            raise Exception(ADB_NOT_FOUND % ('mochitest-remote', b2g_home))
+
         # TODO without os.chdir, chained imports fail below
         os.chdir(self.mochitest_dir)