From af13c2bae84c3152ed1436869cea6c5062a9d9a9 Mon Sep 17 00:00:00 2001 From: James Graham Date: Tue, 25 Aug 2015 14:07:23 +0100 Subject: [PATCH] Bug 1181516 - Allow reftests to take paths to multiple directories containing tests on the command line, r=jmaher This makes reftest command line arguments behave more like other test suites, so we can use a simple unified syntax for e.g. |mach try|. The patch also reworks the command line argument parsing to use argparse rather than optparse, and causes mach to reuse the same parser as the suite. --- layout/tools/reftest/Makefile.in | 1 + layout/tools/reftest/b2g_desktop.py | 23 +- layout/tools/reftest/mach_commands.py | 471 ++++--------- layout/tools/reftest/reftest.js | 62 +- layout/tools/reftest/reftestcommandline.py | 740 +++++++++++++++++++++ layout/tools/reftest/remotereftest.py | 222 +------ layout/tools/reftest/runreftest.py | 334 +++------- layout/tools/reftest/runreftestb2g.py | 254 +------ testing/testsuite-targets.mk | 4 +- 9 files changed, 1077 insertions(+), 1034 deletions(-) create mode 100644 layout/tools/reftest/reftestcommandline.py diff --git a/layout/tools/reftest/Makefile.in b/layout/tools/reftest/Makefile.in index 044ae5fc1a6..f64b2351a26 100644 --- a/layout/tools/reftest/Makefile.in +++ b/layout/tools/reftest/Makefile.in @@ -7,6 +7,7 @@ _DEST_DIR = $(DEPTH)/_tests/reftest _HARNESS_FILES = \ $(srcdir)/runreftest.py \ + $(srcdir)/reftestcommandline.py \ $(srcdir)/remotereftest.py \ $(srcdir)/runreftestb2g.py \ $(srcdir)/b2g_desktop.py \ diff --git a/layout/tools/reftest/b2g_desktop.py b/layout/tools/reftest/b2g_desktop.py index 97a104b58fe..45c0cd6c694 100644 --- a/layout/tools/reftest/b2g_desktop.py +++ b/layout/tools/reftest/b2g_desktop.py @@ -9,7 +9,7 @@ import sys here = os.path.abspath(os.path.dirname(__file__)) -from runreftest import RefTest, ReftestOptions +from runreftest import RefTest from marionette_driver import expected from marionette_driver.by import By @@ -51,12 +51,10 @@ class B2GDesktopReftest(RefTest): f.close() self.marionette.execute_script(self.test_script) - def run_tests(self, test_path, options): - reftestlist = self.getManifestPath(test_path) - if not reftestlist.startswith('file://'): - reftestlist = 'file://%s' % reftestlist + def run_tests(self, tests, options): + manifests = self.resolver.resolveManifests(options, tests) - self.profile = self.create_profile(options, reftestlist, + self.profile = self.create_profile(options, manifests, profile_to_clone=options.profile) env = self.buildBrowserEnv(options, self.profile.profile) kp_kwargs = { 'processOutputLine': [self._on_output], @@ -107,8 +105,8 @@ class B2GDesktopReftest(RefTest): log.info("%s | Running tests: end.", os.path.basename(__file__)) return status - def create_profile(self, options, reftestlist, profile_to_clone=None): - profile = RefTest.createReftestProfile(self, options, reftestlist, + def create_profile(self, options, manifests, profile_to_clone=None): + profile = RefTest.createReftestProfile(self, options, manifests, profile_to_clone=profile_to_clone) prefs = {} @@ -136,7 +134,6 @@ class B2GDesktopReftest(RefTest): prefs["network.dns.localDomains"] = "app://test-container.gaiamobile.org" prefs["reftest.browser.iframe.enabled"] = False prefs["reftest.remote"] = False - prefs["reftest.uri"] = "%s" % reftestlist # Set a future policy version to avoid the telemetry prompt. prefs["toolkit.telemetry.prompted"] = 999 prefs["toolkit.telemetry.notifiedOptOut"] = 999 @@ -192,7 +189,7 @@ class MuletReftest(B2GDesktopReftest): Wait(self.marionette, timeout).until(expected.element_present( By.CSS_SELECTOR, '#homescreen[loading-state=false]')) -def run_desktop_reftests(parser, options, args): +def run_desktop_reftests(parser, options): marionette_args = {} if options.marionette: host, port = options.marionette.split(':') @@ -204,9 +201,7 @@ def run_desktop_reftests(parser, options, args): else: reftest = B2GDesktopReftest(marionette_args) - options = ReftestOptions.verifyCommonOptions(parser, options, reftest) - if options == None: - sys.exit(1) + parser.validate(options, reftest) # add a -bin suffix if b2g-bin exists, but just b2g was specified if options.app[-4:] != '-bin': @@ -219,4 +214,4 @@ def run_desktop_reftests(parser, options, args): if options.desktop and not options.profile: raise Exception("must specify --profile when specifying --desktop") - sys.exit(reftest.run_tests(args[0], options)) + sys.exit(reftest.run_tests(options.tests, options)) diff --git a/layout/tools/reftest/mach_commands.py b/layout/tools/reftest/mach_commands.py index e4b13e5f820..7237e05bfdf 100644 --- a/layout/tools/reftest/mach_commands.py +++ b/layout/tools/reftest/mach_commands.py @@ -23,8 +23,7 @@ from mach.decorators import ( Command, ) - -DEBUGGER_HELP = 'Debugger binary to run test in. Program name or path.' +import reftestcommandline ADB_NOT_FOUND = ''' The %s command requires the adb binary to be on your path. @@ -81,43 +80,10 @@ class ReftestRunner(MozbuildObject): self.tests_dir = os.path.join(self.topobjdir, '_tests') self.reftest_dir = os.path.join(self.tests_dir, 'reftest') - def _manifest_file(self, suite): - """Returns the manifest file used for a given test suite.""" - files = { - 'reftest': 'reftest.list', - 'reftest-ipc': 'reftest.list', - 'crashtest': 'crashtests.list', - 'crashtest-ipc': 'crashtests.list', - 'jstestbrowser': 'jstests.list' - } - assert suite in files - return files[suite] - - def _find_manifest(self, suite, test_file): - """Return a tuple of (manifest-path, filter-string) for running test_file. - - test_file can be a relative path to a single test file or manifest from - the top source directory, an absolute path to the same, or a directory - containing a manifest. - """ - assert test_file - path_arg = self._wrap_path_argument(test_file) - relpath = path_arg.relpath() - - if os.path.isdir(path_arg.srcdir_path()): - return (mozpath.join(relpath, self._manifest_file(suite)), None) - - if relpath.endswith('.list'): - return (relpath, None) - - return (self._find_manifest(suite, mozpath.dirname(test_file))[0], - mozpath.basename(test_file)) - def _make_shell_string(self, s): return "'%s'" % re.sub("'", r"'\''", s) - def run_b2g_test(self, b2g_home=None, xre_path=None, test_file=None, - suite=None, filter=None, **kwargs): + def run_b2g_test(self, b2g_home=None, xre_path=None, **kwargs): """Runs a b2g reftest. filter is a regular expression (in JS syntax, as could be passed to the @@ -130,324 +96,175 @@ class ReftestRunner(MozbuildObject): suite is the type of reftest to run. It can be one of ('reftest', 'crashtest'). """ - if suite not in ('reftest', 'crashtest'): + if kwargs["suite"] not in ('reftest', 'crashtest'): raise Exception('None or unrecognized reftest suite type.') + sys.path.insert(0, self.reftest_dir) + + test_subdir = {"reftest": os.path.join('layout', 'reftests'), + "crashtest": os.path.join('layout', 'crashtest')}[kwargs["suite"]] + # Find the manifest file - if not test_file: - if suite == 'reftest': - test_file = mozpath.join('layout', 'reftests') - elif suite == 'crashtest': - test_file = mozpath.join('testing', 'crashtest') + if not kwargs["tests"]: + if not os.path.exists(os.path.join(self.topsrcdir, test_subdir)): + test_file = mozpath.relpath(os.path.abspath(test_subdir), + self.topsrcdir) + kwargs["tests"] = [test_subdir] - if not os.path.exists(os.path.join(self.topsrcdir, test_file)): - test_file = mozpath.relpath(os.path.abspath(test_file), - self.topsrcdir) - - (manifest, single_file_filter) = self._find_manifest(suite, test_file) - if not os.path.exists(mozpath.join(self.topsrcdir, manifest)): - raise Exception('No manifest file was found at %s.' % manifest) - if single_file_filter: - if filter: - raise Exception('Cannot run single files in conjunction with --filter') - filter = single_file_filter - - # Need to chdir to reftest_dir otherwise imports fail below. - os.chdir(self.reftest_dir) - - # The imp module can spew warnings if the modules below have - # already been imported, ignore them. - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - - import imp - path = os.path.join(self.reftest_dir, 'runreftestb2g.py') - with open(path, 'r') as fh: - imp.load_module('reftest', fh, path, ('.py', 'r', imp.PY_SOURCE)) - import reftest - - # Set up the reftest options. - parser = reftest.B2GOptions() - options, args = parser.parse_args([]) - - # Tests need to be served from a subdirectory of the server. Symlink - # topsrcdir here to get around this. tests = os.path.join(self.reftest_dir, 'tests') if not os.path.isdir(tests): os.symlink(self.topsrcdir, tests) - args.insert(0, os.path.join('tests', manifest)) - for k, v in kwargs.iteritems(): - setattr(options, k, v) + for i, path in enumerate(kwargs["tests"]): + # Non-absolute paths are relative to the packaged directory, which + # has an extra tests/ at the start + if os.path.exists(os.path.abspath(path)): + path = os.path.relpath(path, os.path.join(self.topsrcdir)) + kwargs["tests"][i] = os.path.join('tests', path) if conditions.is_b2g_desktop(self): - if self.substs.get('ENABLE_MARIONETTE') != '1': - print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop', - self.mozconfig['path'])) - return 1 + return self.run_b2g_desktop(**kwargs) - options.profile = options.profile or os.environ.get('GAIA_PROFILE') - if not options.profile: + return self.run_b2g_remote(b2g_home, xre_path, **kwargs) + + def run_b2g_desktop(self, **kwargs): + if self.substs.get('ENABLE_MARIONETTE') != '1': + print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop', + self.mozconfig['path'])) + return 1 + + if not kwargs["profile"]: + gaia_profile = os.environ.get('GAIA_PROFILE') + if not gaia_profile: print(GAIA_PROFILE_NOT_FOUND % 'reftest-b2g-desktop') return 1 + kwargs["profile"] = gaia_profile - if os.path.isfile(os.path.join(options.profile, 'extensions', \ - 'httpd@gaiamobile.org')): - print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop', - options.profile)) - return 1 - options.desktop = True - options.app = self.get_binary_path() - if options.oop: - options.browser_arg = '-oop' - if not options.app.endswith('-bin'): - options.app = '%s-bin' % options.app - if not os.path.isfile(options.app): - options.app = options.app[:-len('-bin')] + if os.path.isfile(os.path.join(kwargs["profile"], 'extensions', + 'httpd@gaiamobile.org')): + print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop', + kwargs["profile"])) + return 1 - return reftest.run_desktop_reftests(parser, options, args) + kwargs["desktop"] = True + kwargs["app"] = self.get_binary_path() + if kwargs["oop"]: + options.browser_arg = '-oop' + if not kwargs["app"].endswith('-bin'): + kwargs["app"] = '%s-bin' % options.app + if not os.path.isfile(kwargs["app"]): + options.app = kwargs["app"][:-len('-bin')] + return runreftestb2g.run(**kwargs) + + def run_b2g_remote(self, b2g_home, xre_path, **kwargs): + import runreftestb2g try: which.which('adb') except which.WhichError: # TODO Find adb automatically if it isn't on the path - raise Exception(ADB_NOT_FOUND % ('%s-remote' % suite, b2g_home)) + raise Exception(ADB_NOT_FOUND % ('%s-remote' % kwargs["suite"], b2g_home)) - options.b2gPath = b2g_home - options.logdir = self.reftest_dir - options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver') - options.xrePath = xre_path - options.ignoreWindowSize = True - options.filter = filter + kwargs["b2gPath"] = b2g_home + kwargs["logdir"] = self.reftest_dir + kwargs["httpdPath"] = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver') + kwargs["xrePath"] = xre_path + kwargs["ignoreWindowSize"] = True # Don't enable oop for crashtest until they run oop in automation - if suite == 'reftest': - options.oop = True + if kwargs["suite"] == 'reftest': + kwargs["oop"] = True - return reftest.run_remote_reftests(parser, options, args) + return runreftestb2g.run_remote(**kwargs) - def run_desktop_test(self, test_file=None, filter=None, suite=None, - debugger=None, debugger_args=None, parallel=False, shuffle=False, - e10s=False, extraPrefs=None, this_chunk=None, total_chunks=None): - """Runs a reftest. + def run_desktop_test(self, **kwargs): + """Runs a reftest.""" + import runreftest - test_file is a path to a test file. It can be a relative path from the - top source directory, an absolute filename, or a directory containing - test files. - - filter is a regular expression (in JS syntax, as could be passed to the - RegExp constructor) to select which reftests to run from the manifest. - - suite is the type of reftest to run. It can be one of ('reftest', - 'crashtest', 'jstestbrowser'). - - debugger is the program name (in $PATH) or the full path of the - debugger to run. - - debugger_args are the arguments passed to the debugger. - - parallel indicates whether tests should be run in parallel or not. - - shuffle indicates whether to run tests in random order. - """ - - if suite not in ('reftest', 'reftest-ipc', 'crashtest', 'crashtest-ipc', 'jstestbrowser'): + if kwargs["suite"] not in ('reftest', 'crashtest', 'jstestbrowser'): raise Exception('None or unrecognized reftest suite type.') - env = {} - extra_args = [] + default_manifest = { + "reftest": (self.topsrcdir, "layout", "reftests", "reftest.list"), + "crashtest": (self.topsrcdir, "testing", "crashtest", "crashtests.list"), + "jstestbrowser": (self.topobjdir, "dist", "test-stage", "jsreftest", "tests", + "jstests.list") + } - if test_file: - (path, single_file_filter) = self._find_manifest(suite, test_file) - if not os.path.exists(mozpath.join(self.topsrcdir, path)): - raise Exception('No manifest file was found at %s.' % path) - if single_file_filter: - if filter: - raise Exception('Cannot run single files in conjunction with --filter') - filter = single_file_filter - env[b'TEST_PATH'] = path - if filter: - extra_args.extend(['--filter', self._make_shell_string(filter)]) + kwargs["extraProfileFiles"] = [os.path.join(self.topobjdir, "dist", "plugins")] + kwargs["symbolsPath"] = os.path.join(self.topobjdir, "crashreporter-symbols") - pass_thru = False + if not kwargs["tests"]: + kwargs["tests"] = [os.path.join(*default_manifest[kwargs["suite"]])] - if debugger: - extra_args.append('--debugger=\'%s\'' % debugger) - pass_thru = True - if debugger_args: - # Use _make_shell_string (which quotes) so that we - # handle multiple args being passed to the debugger. - extra_args.extend(['--debugger-args', self._make_shell_string(debugger_args)]) - else: - if debugger_args: - print("--debugger-args passed, but no debugger specified.") - return 1 + if kwargs["suite"] == "jstestbrowser": + kwargs["extraProfileFiles"].append(os.path.join(self.topobjdir, "dist", + "test-stage", "jsreftest", + "tests", "user.js")) - if parallel: - extra_args.append('--run-tests-in-parallel') + if not kwargs["runTestsInParallel"]: + kwargs["logFile"] = "%s.log" % kwargs["suite"] - if shuffle: - extra_args.append('--shuffle') - - if e10s: - extra_args.append('--e10s') - - if extraPrefs: - for pref in extraPrefs: - extra_args.extend(['--setpref', pref]) - - if this_chunk: - extra_args.append('--this-chunk=%s' % this_chunk) - - if total_chunks: - extra_args.append('--total-chunks=%s' % total_chunks) - - if extra_args: - args = [os.environ.get(b'EXTRA_TEST_ARGS', '')] - args.extend(extra_args) - env[b'EXTRA_TEST_ARGS'] = ' '.join(args) - - # TODO hook up harness via native Python - return self._run_make(directory='.', target=suite, append_env=env, - pass_thru=pass_thru, ensure_exit_code=False) - - -def ReftestCommand(func): - """Decorator that adds shared command arguments to reftest commands.""" - - debugger = CommandArgument('--debugger', metavar='DEBUGGER', - help=DEBUGGER_HELP) - func = debugger(func) - - debugger_args = CommandArgument('--debugger-args', metavar='DEBUGGER_ARGS', - help='Arguments to pass to the debugger.') - func = debugger_args(func) - - flter = CommandArgument('--filter', metavar='REGEX', - help='A JS regular expression to match test URLs against, to select ' - 'a subset of tests to run.') - func = flter(func) - - path = CommandArgument('test_file', nargs='?', metavar='MANIFEST', - help='Reftest manifest file, or a directory in which to select ' - 'reftest.list. If omitted, the entire test suite is executed.') - func = path(func) - - parallel = CommandArgument('--parallel', action='store_true', - help='Run tests in parallel.') - func = parallel(func) - - shuffle = CommandArgument('--shuffle', action='store_true', - help='Run tests in random order.') - func = shuffle(func) - - e10s = CommandArgument('--e10s', action='store_true', - help='Use content processes.') - func = e10s(func) - - extraPrefs = CommandArgument('--setpref', action='append', - default=[], dest='extraPrefs', metavar='PREF=VALUE', - help='Set prefs in the reftest profile.') - func = extraPrefs(func) - - totalChunks = CommandArgument('--total-chunks', - help = 'How many chunks to split the tests up into.') - func = totalChunks(func) - - thisChunk = CommandArgument('--this-chunk', - help = 'Which chunk to run between 1 and --total-chunks.') - func = thisChunk(func) - - return func - -def B2GCommand(func): - """Decorator that adds shared command arguments to b2g reftest commands.""" - - busybox = CommandArgument('--busybox', default=None, - help='Path to busybox binary to install on device') - func = busybox(func) - - logdir = CommandArgument('--logdir', default=None, - help='directory to store log files') - func = logdir(func) - - sdcard = CommandArgument('--sdcard', default="10MB", - help='Define size of sdcard: 1MB, 50MB...etc') - func = sdcard(func) - - emulator_res = CommandArgument('--emulator-res', default='800x1000', - help='Emulator resolution of the format \'x\'') - func = emulator_res(func) - - marionette = CommandArgument('--marionette', default=None, - help='host:port to use when connecting to Marionette') - func = marionette(func) - - totalChunks = CommandArgument('--total-chunks', dest='totalChunks', - type = int, - help = 'How many chunks to split the tests up into.') - func = totalChunks(func) - - thisChunk = CommandArgument('--this-chunk', dest='thisChunk', - type = int, - help = 'Which chunk to run between 1 and --total-chunks.') - func = thisChunk(func) - - flter = CommandArgument('--filter', metavar='REGEX', - help='A JS regular expression to match test URLs against, to select ' - 'a subset of tests to run.') - func = flter(func) - - oop = CommandArgument('--enable-oop', action='store_true', dest='oop', - help = 'Run tests in out-of-process mode.') - func = oop(func) - - path = CommandArgument('test_file', default=None, nargs='?', - metavar='TEST', - help='Test to run. Can be specified as a single file, a ' \ - 'directory, or omitted. If omitted, the entire test suite is ' \ - 'executed.') - func = path(func) - - return func + #Remove the stdout handler from the internal logger and let mach deal with it + runreftest.log.removeHandler(runreftest.log.handlers[0]) + self.log_manager.enable_unstructured() + rv = runreftest.run(**kwargs) + self.log_manager.disable_unstructured() + return rv @CommandProvider class MachCommands(MachCommandBase): - @Command('reftest', category='testing', description='Run reftests (layout and graphics correctness).') - @ReftestCommand - def run_reftest(self, test_file, **kwargs): - return self._run_reftest(test_file, suite='reftest', **kwargs) + @Command('reftest', + category='testing', + description='Run reftests (layout and graphics correctness).', + parser=reftestcommandline.DesktopArgumentsParser) + def run_reftest(self, **kwargs): + kwargs["suite"] = "reftest" + return self._run_reftest(**kwargs) - @Command('jstestbrowser', category='testing', - description='Run js/src/tests in the browser.') - @ReftestCommand - def run_jstestbrowser(self, test_file, **kwargs): - return self._run_reftest(test_file, suite='jstestbrowser', **kwargs) + @Command('jstestbrowser', + category='testing', + description='Run js/src/tests in the browser.', + parser=reftestcommandline.DesktopArgumentsParser) + def run_jstestbrowser(self, **kwargs): + self._mach_context.commands.dispatch("build", + self._mach_context, + what=["stage-jstests"]) + kwargs["suite"] = "jstestbrowser" + return self._run_reftest(**kwargs) - @Command('reftest-ipc', category='testing', - description='Run IPC reftests (layout and graphics correctness, separate process).') - @ReftestCommand - def run_ipc(self, test_file, **kwargs): - return self._run_reftest(test_file, suite='reftest-ipc', **kwargs) + @Command('reftest-ipc', + category='testing', + description='Run IPC reftests (layout and graphics correctness, separate process).', + parser=reftestcommandline.DesktopArgumentsParser) + def run_ipc(self, **kwargs): + kwargs["ipc"] = True + kwargs["suite"] = "reftest" + return self._run_reftest(**kwargs) - @Command('crashtest', category='testing', - description='Run crashtests (Check if crashes on a page).') - @ReftestCommand - def run_crashtest(self, test_file, **kwargs): - return self._run_reftest(test_file, suite='crashtest', **kwargs) + @Command('crashtest', + category='testing', + description='Run crashtests (Check if crashes on a page).', + parser=reftestcommandline.DesktopArgumentsParser) + def run_crashtest(self, **kwargs): + kwargs["suite"] = "crashtest" + return self._run_reftest(**kwargs) - @Command('crashtest-ipc', category='testing', - description='Run IPC crashtests (Check if crashes on a page, separate process).') - @ReftestCommand - def run_crashtest_ipc(self, test_file, **kwargs): - return self._run_reftest(test_file, suite='crashtest-ipc', **kwargs) + @Command('crashtest-ipc', + category='testing', + description='Run IPC crashtests (Check if crashes on a page, separate process).', + parser=reftestcommandline.DesktopArgumentsParser) + def run_crashtest_ipc(self, **kwargs): + kwargs["ipc"] = True + kwargs["suite"] = "crashtest" + return self._run_reftest(**kwargs) - def _run_reftest(self, test_file=None, suite=None, **kwargs): + def _run_reftest(self, **kwargs): reftest = self._spawn(ReftestRunner) - return reftest.run_desktop_test(test_file, suite=suite, **kwargs) + return reftest.run_desktop_test(**kwargs) # TODO For now b2g commands will only work with the emulator, @@ -466,27 +283,30 @@ class B2GCommands(MachCommandBase): setattr(self, attr, getattr(context, attr, None)) @Command('reftest-remote', category='testing', - description='Run a remote reftest (b2g layout and graphics correctness, remote device).', - conditions=[conditions.is_b2g, is_emulator]) - @B2GCommand - def run_reftest_remote(self, test_file, **kwargs): - return self._run_reftest(test_file, suite='reftest', **kwargs) + description='Run a remote reftest (b2g layout and graphics correctness, remote device).', + conditions=[conditions.is_b2g, is_emulator], + parser=reftestcommandline.B2GArgumentParser) + def run_reftest_remote(self, **kwargs): + kwargs["suite"] = "reftest" + return self._run_reftest(**kwargs) @Command('reftest-b2g-desktop', category='testing', - description='Run a b2g desktop reftest (b2g desktop layout and graphics correctness).', - conditions=[conditions.is_b2g_desktop]) - @B2GCommand - def run_reftest_b2g_desktop(self, test_file, **kwargs): - return self._run_reftest(test_file, suite='reftest', **kwargs) + description='Run a b2g desktop reftest (b2g desktop layout and graphics correctness).', + conditions=[conditions.is_b2g_desktop], + parser=reftestcommandline.B2GArgumentParser) + def run_reftest_b2g_desktop(self, **kwargs): + kwargs["suite"] = "reftest" + return self._run_reftest(**kwargs) @Command('crashtest-remote', category='testing', - description='Run a remote crashtest (Check if b2g crashes on a page, remote device).', - conditions=[conditions.is_b2g, is_emulator]) - @B2GCommand + description='Run a remote crashtest (Check if b2g crashes on a page, remote device).', + conditions=[conditions.is_b2g, is_emulator], + parser=reftestcommandline.B2GArgumentParser) def run_crashtest_remote(self, test_file, **kwargs): - return self._run_reftest(test_file, suite='crashtest', **kwargs) + kwargs["suite"] = "crashtest" + return self._run_reftest(**kwargs) - def _run_reftest(self, test_file=None, suite=None, **kwargs): + def _run_reftest(self, **kwargs): if self.device_name: if self.device_name.startswith('emulator'): emulator = 'arm' @@ -495,5 +315,4 @@ class B2GCommands(MachCommandBase): kwargs['emulator'] = emulator reftest = self._spawn(ReftestRunner) - return reftest.run_b2g_test(self.b2g_home, self.xre_path, - test_file, suite=suite, **kwargs) + return reftest.run_b2g_test(self.b2g_home, self.xre_path, **kwargs) diff --git a/layout/tools/reftest/reftest.js b/layout/tools/reftest/reftest.js index b8b5ba73219..ca61a086b01 100644 --- a/layout/tools/reftest/reftest.js +++ b/layout/tools/reftest/reftest.js @@ -48,7 +48,7 @@ var gShuffle = false; var gTotalChunks = 0; var gThisChunk = 0; var gContainingWindow = null; -var gURLFilterRegex = null; +var gURLFilterRegex = {}; const FOCUS_FILTER_ALL_TESTS = "all"; const FOCUS_FILTER_NEEDS_FOCUS_TESTS = "needs-focus"; const FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS = "non-needs-focus"; @@ -389,10 +389,6 @@ function InitAndStartRefTests() gThisChunk = 0; } - try { - gURLFilterRegex = new RegExp(prefs.getCharPref("reftest.filter")); - } catch(e) {} - try { gFocusFilterMode = prefs.getCharPref("reftest.focusFilterMode"); } catch(e) {} @@ -449,7 +445,7 @@ function Shuffle(array) function StartTests() { - var uri; + var manifests; /* These prefs are optional, so we don't need to spit an error to the log */ try { var prefs = Components.classes["@mozilla.org/preferences-service;1"]. @@ -476,23 +472,29 @@ function StartTests() gRunSlowTests = false; } - try { - uri = prefs.getCharPref("reftest.uri"); - } catch(e) { - uri = ""; - } - - if (uri == "") { - gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | Unable to find reftest.uri pref. Please ensure your profile is setup properly\n"); - DoneTests(); - } - if (gShuffle) { gNoCanvasCache = true; } + gURLs = []; + try { - ReadTopManifest(uri); + var manifests = JSON.parse(prefs.getCharPref("reftest.manifests")); + gURLFilterRegex = manifests[null]; + } catch(e) { + gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | Unable to find reftest.manifests pref. Please ensure your profile is setup properly\n"); + DoneTests(); + } + + try { + var globalFilter = manifests.hasOwnProperty("") ? new RegExp(manifests[""]) : null; + var manifestURLs = Object.keys(manifests); + manifestURLs.sort(); + manifestURLs.forEach(function(manifestURL) { + gDumpLog("Readings manifest" + manifestURL + "\n"); + var pathFilters = manifests[manifestURL].map(function(x) {return new RegExp(x)}); + ReadTopManifest(manifestURL, [globalFilter, pathFilters]); + }); BuildUseCounts(); // Filter tests which will be skipped to get a more even distribution when chunking @@ -745,18 +747,22 @@ function AddPrefSettings(aWhere, aPrefName, aPrefValExpression, aSandbox, aTestP return true; } -function ReadTopManifest(aFileURL) +function ReadTopManifest(aFileURL, aFilters) { - gURLs = new Array(); var url = gIOService.newURI(aFileURL, null, null); if (!url) throw "Expected a file or http URL for the manifest."; - ReadManifest(url, EXPECTED_PASS); + ReadManifest(url, EXPECTED_PASS, aFilters); } -function AddTestItem(aTest) +function AddTestItem(aTest, aFilters) { - if (gURLFilterRegex && !gURLFilterRegex.test(aTest.url1.spec)) + if (!aFilters) + aFilters = [null, []]; + + if ((aFilters[0] && !aFilters[0].test(aTest.url1.spec)) || + (aFilters[1].length > 0 && + !aFilters[1].some(function(filter) {return filter.test(aTest.url1.spec)}))) return; if (gFocusFilterMode == FOCUS_FILTER_NEEDS_FOCUS_TESTS && !aTest.needsFocus) @@ -769,7 +775,7 @@ function AddTestItem(aTest) // Note: If you materially change the reftest manifest parsing, // please keep the parser in print-manifest-dirs.py in sync. -function ReadManifest(aURL, inherited_status) +function ReadManifest(aURL, inherited_status, aFilters) { var secMan = CC[NS_SCRIPTSECURITYMANAGER_CONTRACTID] .getService(CI.nsIScriptSecurityManager); @@ -986,7 +992,7 @@ function ReadManifest(aURL, inherited_status) var incURI = gIOService.newURI(items[1], null, listURL); secMan.checkLoadURIWithPrincipal(principal, incURI, CI.nsIScriptSecurityManager.DISALLOW_SCRIPT); - ReadManifest(incURI, expected_status); + ReadManifest(incURI, expected_status, aFilters); } else if (items[0] == TYPE_LOAD) { if (items.length != 2) throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to load"; @@ -1016,7 +1022,7 @@ function ReadManifest(aURL, inherited_status) fuzzyMaxPixels: fuzzy_max_pixels, url1: testURI, url2: null, - chaosMode: chaosMode }); + chaosMode: chaosMode }, aFilters); } else if (items[0] == TYPE_SCRIPT) { if (items.length != 2) throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to script"; @@ -1043,7 +1049,7 @@ function ReadManifest(aURL, inherited_status) fuzzyMaxPixels: fuzzy_max_pixels, url1: testURI, url2: null, - chaosMode: chaosMode }); + chaosMode: chaosMode }, aFilters); } else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL) { if (items.length != 3) throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to " + items[0]; @@ -1073,7 +1079,7 @@ function ReadManifest(aURL, inherited_status) fuzzyMaxPixels: fuzzy_max_pixels, url1: testURI, url2: refURI, - chaosMode: chaosMode }); + chaosMode: chaosMode }, aFilters); } else { throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": unknown test type " + items[0]; } diff --git a/layout/tools/reftest/reftestcommandline.py b/layout/tools/reftest/reftestcommandline.py new file mode 100644 index 00000000000..b258f9d4862 --- /dev/null +++ b/layout/tools/reftest/reftestcommandline.py @@ -0,0 +1,740 @@ +import argparse +import os +from collections import OrderedDict +from urlparse import urlparse + +here = os.path.abspath(os.path.dirname(__file__)) + + +class ReftestArgumentsParser(argparse.ArgumentParser): + def __init__(self, **kwargs): + super(ReftestArgumentsParser, self).__init__(**kwargs) + + # Try to import a MozbuildObject. Success indicates that we are + # running from a source tree. This allows some defaults to be set + # from the source tree. + try: + from mozbuild.base import MozbuildObject + self.build_obj = MozbuildObject.from_environment(cwd=here) + except ImportError: + self.build_obj = None + + self.add_argument("--xre-path", + action="store", + type=str, + dest="xrePath", + # individual scripts will set a sane default + default=None, + help="absolute path to directory containing XRE (probably xulrunner)") + + self.add_argument("--symbols-path", + action="store", + type=str, + dest="symbolsPath", + default=None, + help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols") + + self.add_argument("--debugger", + action="store", + dest="debugger", + help="use the given debugger to launch the application") + + self.add_argument("--debugger-args", + action="store", + dest="debuggerArgs", + help="pass the given args to the debugger _before_ " + "the application on the command line") + + self.add_argument("--debugger-interactive", + action="store_true", + dest="debuggerInteractive", + help="prevents the test harness from redirecting " + "stdout and stderr for interactive debuggers") + + self.add_argument("--appname", + action="store", + type=str, + dest="app", + default=None, + help="absolute path to application, overriding default") + + self.add_argument("--extra-profile-file", + action="append", + dest="extraProfileFiles", + default=[], + help="copy specified files/dirs to testing profile") + + self.add_argument("--timeout", + action="store", + dest="timeout", + type=int, + default=5 * 60, # 5 minutes per bug 479518 + help="reftest will timeout in specified number of seconds. [default %(default)s].") + + self.add_argument("--leak-threshold", + action="store", + type=int, + dest="defaultLeakThreshold", + default=0, + help="fail if the number of bytes leaked in default " + "processes through refcounted objects (or bytes " + "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) " + "is greater than the given number") + + self.add_argument("--utility-path", + action="store", + type=str, + dest="utilityPath", + default="bindir", + help="absolute path to directory containing utility " + "programs (xpcshell, ssltunnel, certutil)") + + self.add_argument("--total-chunks", + type=int, + dest="totalChunks", + help="how many chunks to split the tests up into") + + self.add_argument("--this-chunk", + type=int, + dest="thisChunk", + help="which chunk to run between 1 and --total-chunks") + + self.add_argument("--log-file", + action="store", + type=str, + dest="logFile", + default=None, + help="file to log output to in addition to stdout") + + self.add_argument("--skip-slow-tests", + dest="skipSlowTests", + action="store_true", + default=False, + help="skip tests marked as slow when running") + + self.add_argument("--ignore-window-size", + dest="ignoreWindowSize", + action="store_true", + default=False, + help="ignore the window size, which may cause spurious failures and passes") + + self.add_argument("--install-extension", + action="append", + dest="extensionsToInstall", + default=[], + help="install the specified extension in the testing profile. " + "The extension file's name should be .xpi where is " + "the extension's id as indicated in its install.rdf. " + "An optional path can be specified too.") + + self.add_argument("--setenv", + action="append", + type=str, + default=[], + dest="environment", + metavar="NAME=VALUE", + help="sets the given variable in the application's " + "environment") + + self.add_argument("--filter", + action="store", + type=str, + dest="filter", + help="specifies a regular expression (as could be passed to the JS " + "RegExp constructor) to test against URLs in the reftest manifest; " + "only test items that have a matching test URL will be run.") + + self.add_argument("--shuffle", + action="store_true", + default=False, + dest="shuffle", + help="run reftests in random order") + + self.add_argument("--focus-filter-mode", + action="store", + type=str, + dest="focusFilterMode", + default="all", + help="filters tests to run by whether they require focus. " + "Valid values are `all', `needs-focus', or `non-needs-focus'. " + "Defaults to `all'.") + + self.add_argument("--e10s", + action="store_true", + default=False, + dest="e10s", + help="enables content processes") + + self.add_argument("--setpref", + action="append", + type=str, + default=[], + dest="extraPrefs", + metavar="PREF=VALUE", + help="defines an extra user preference") + + self.add_argument("--reftest-extension-path", + action="store", + dest="reftestExtensionPath", + help="Path to the reftest extension") + + self.add_argument("--special-powers-extension-path", + action="store", + dest="specialPowersExtensionPath", + help="Path to the special powers extension") + + self.add_argument("--suite", + choices=["reftest", "crashtest", "jstestbrowser"], + default=None, + help=argparse.SUPPRESS) + + self.add_argument("tests", + metavar="TEST_PATH", + nargs="*", + help="Path to test file, manifest file, or directory containing tests") + + def get_ip(self): + import moznetwork + if os.name != "nt": + return moznetwork.get_ip() + else: + self.error( + "ERROR: you must specify a --remote-webserver=\n") + + def set_default_suite(self, options): + manifests = OrderedDict([("reftest.list", "reftest"), + ("crashtests.list", "crashtest"), + ("jstests.list", "jstestbrowser")]) + + for test_path in options.tests: + file_name = os.path.basename(test_path) + if file_name in manifests: + options.suite = manifests[file_name] + return + + for test_path in options.tests: + for manifest_file, suite in manifests.iteritems(): + if os.path.exists(os.path.join(test_path, manifest_file)): + options.suite = suite + return + + self.error("Failed to determine test suite; supply --suite to set this explicitly") + + def validate(self, options, reftest): + import sys + + if not options.tests: + # Can't just set this in the argument parser because mach will set a default + self.error("Must supply at least one path to a manifest file, test directory, or test file to run.") + + if options.suite is None: + self.set_default_suite(options) + + if options.totalChunks is not None and options.thisChunk is None: + self.error( + "thisChunk must be specified when totalChunks is specified") + + if options.totalChunks: + if not 1 <= options.thisChunk <= options.totalChunks: + self.error("thisChunk must be between 1 and totalChunks") + + if options.logFile: + options.logFile = reftest.getFullPath(options.logFile) + + if options.xrePath is not None: + if not os.access(options.xrePath, os.F_OK): + self.error("--xre-path '%s' not found" % options.xrePath) + if not os.path.isdir(options.xrePath): + self.error("--xre-path '%s' is not a directory" % + options.xrePath) + options.xrePath = reftest.getFullPath(options.xrePath) + + if options.reftestExtensionPath is None: + if self.build_obj is not None: + options.reftestExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests", + "reftest", "reftest") + else: + options.reftestExtensionPath = os.path.join(here, "reftest") + + if (options.specialPowersExtensionPath is None and + options.suite in ["crashtest", "jstestbrowser"]): + if self.build_obj is not None: + options.specialPowersExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests", + "reftest", "specialpowers") + else: + options.specialPowersExtensionPath = os.path.join( + here, "specialpowers") + + options.leakThresholds = { + "default": options.defaultLeakThreshold, + "tab": 5000, # See dependencies of bug 1051230. + } + + +class DesktopArgumentsParser(ReftestArgumentsParser): + def __init__(self, **kwargs): + super(DesktopArgumentsParser, self).__init__(**kwargs) + + self.add_argument("--run-tests-in-parallel", + action="store_true", + default=False, + dest="runTestsInParallel", + help="run tests in parallel if possible") + + self.add_argument("--ipc", + action="store_true", + default=False, + help="Run in out-of-processes mode") + + def _prefs_oop(self): + import mozinfo + prefs = ["layers.async-pan-zoom.enabled=true", + "browser.tabs.remote.autostart=true"] + if mozinfo.os == "win": + prefs.append("layers.acceleration.disabled=true") + + return prefs + + def _prefs_gpu(self): + if mozinfo.os != "win": + return ["layers.acceleration.force-enabled=true"] + return [] + + def validate(self, options, reftest): + super(DesktopArgumentsParser, self).validate(options, reftest) + + if options.ipc: + for item in self._prefs_oop(): + if item not in options.extraPrefs: + options.extraPrefs.append(item) + + if options.runTestsInParallel: + if options.logFile is not None: + self.error("cannot specify logfile with parallel tests") + if options.totalChunks is not None or options.thisChunk is not None: + self.error( + "cannot specify thisChunk or totalChunks with parallel tests") + if options.focusFilterMode != "all": + self.error("cannot specify focusFilterMode with parallel tests") + if options.debugger is not None: + self.error("cannot specify a debugger with parallel tests") + + if not options.tests: + self.error("No test files specified.") + + if options.app is None: + bin_dir = (self.build_obj.get_binary_path() if + self.build_obj and self.build_obj.substs[ + 'MOZ_BUILD_APP'] != 'mobile/android' + else None) + + if bin_dir: + options.app = bin_dir + else: + self.error( + "could not find the application path, --appname must be specified") + + options.app = reftest.getFullPath(options.app) + if not os.path.exists(options.app): + self.error("""Error: Path %(app)s doesn't exist. + Are you executing $objdir/_tests/reftest/runreftest.py?""" + % {"app": options.app}) + + if options.xrePath is None: + options.xrePath = os.path.dirname(options.app) + + if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: + options.symbolsPath = reftest.getFullPath(options.symbolsPath) + + options.utilityPath = reftest.getFullPath(options.utilityPath) + + +class B2GArgumentParser(ReftestArgumentsParser): + def __init__(self, **kwargs): + super(B2GArgumentParser, self).__init__(**kwargs) + + self.add_argument("--browser-arg", + action="store", + type=str, + dest="browser_arg", + help="Optional command-line arg to pass to the browser") + + self.add_argument("--b2gpath", + action="store", + type=str, + dest="b2gPath", + help="path to B2G repo or qemu dir") + + self.add_argument("--marionette", + action="store", + type=str, + dest="marionette", + help="host:port to use when connecting to Marionette") + + self.add_argument("--emulator", + action="store", + type=str, + dest="emulator", + help="Architecture of emulator to use: x86 or arm") + + self.add_argument("--emulator-res", + action="store", + type=str, + dest="emulator_res", + help="Emulator resolution of the format 'x'") + + self.add_argument("--no-window", + action="store_true", + dest="noWindow", + default=False, + help="Pass --no-window to the emulator") + + self.add_argument("--adbpath", + action="store", + type=str, + dest="adb_path", + default="adb", + help="path to adb") + + self.add_argument("--deviceIP", + action="store", + type=str, + dest="deviceIP", + help="ip address of remote device to test") + + self.add_argument("--devicePort", + action="store", + type=str, + dest="devicePort", + default="20701", + help="port of remote device to test") + + self.add_argument("--remote-logfile", + action="store", + type=str, + dest="remoteLogFile", + help="Name of log file on the device relative to the device root. PLEASE ONLY USE A FILENAME.") + + self.add_argument("--remote-webserver", + action="store", + type=str, + dest="remoteWebServer", + help="ip address where the remote web server is hosted at") + + self.add_argument("--http-port", + action="store", + type=str, + dest="httpPort", + help="ip address where the remote web server is hosted at") + + self.add_argument("--ssl-port", + action="store", + type=str, + dest="sslPort", + help="ip address where the remote web server is hosted at") + + self.add_argument("--pidfile", + action="store", + type=str, + dest="pidFile", + default="", + help="name of the pidfile to generate") + + self.add_argument("--gecko-path", + action="store", + type=str, + dest="geckoPath", + help="the path to a gecko distribution that should " + "be installed on the emulator prior to test") + + self.add_argument("--logdir", + action="store", + type=str, + dest="logdir", + help="directory to store log files") + + self.add_argument('--busybox', + action='store', + type=str, + dest='busybox', + help="Path to busybox binary to install on device") + + self.add_argument("--httpd-path", + action="store", + type=str, + dest="httpdPath", + help="path to the httpd.js file") + + self.add_argument("--profile", + action="store", + type=str, + dest="profile", + help="for desktop testing, the path to the " + "gaia profile to use") + + self.add_argument("--desktop", + action="store_true", + dest="desktop", + default=False, + help="Run the tests on a B2G desktop build") + + self.add_argument("--mulet", + action="store_true", + dest="mulet", + default=False, + help="Run the tests on a B2G desktop build") + + self.add_argument("--enable-oop", + action="store_true", + dest="oop", + default=False, + help="Run the tests out of process") + + self.set_defaults(remoteTestRoot=None, + logFile="reftest.log", + autorun=True, + closeWhenDone=True, + testPath="") + + def validate_remote(self, options, automation): + if not options.app: + options.app = automation.DEFAULT_APP + + if not options.remoteTestRoot: + options.remoteTestRoot = automation._devicemanager.deviceRoot + \ + "/reftest" + + options.remoteProfile = options.remoteTestRoot + "/profile" + + productRoot = options.remoteTestRoot + "/" + automation._product + if options.utilityPath is None: + options.utilityPath = productRoot + "/bin" + + if not options.httpPort: + options.httpPort = automation.DEFAULT_HTTP_PORT + + if not options.sslPort: + options.sslPort = automation.DEFAULT_SSL_PORT + + if options.remoteWebServer is None: + options.remoteWebServer = self.get_ip() + + options.webServer = options.remoteWebServer + + if options.geckoPath and not options.emulator: + self.error( + "You must specify --emulator if you specify --gecko-path") + + if options.logdir and not options.emulator: + self.error("You must specify --emulator if you specify --logdir") + + if options.remoteLogFile is None: + options.remoteLogFile = "reftest.log" + + options.localLogName = options.remoteLogFile + options.remoteLogFile = options.remoteTestRoot + \ + '/' + options.remoteLogFile + + # Ensure that the options.logfile (which the base class uses) is set to + # the remote setting when running remote. Also, if the user set the + # log file name there, use that instead of reusing the remotelogfile as + # above. + if (options.logFile): + # If the user specified a local logfile name use that + options.localLogName = options.logFile + options.logFile = options.remoteLogFile + + # Only reset the xrePath if it wasn't provided + if options.xrePath is None: + options.xrePath = options.utilityPath + options.xrePath = os.path.abspath(options.xrePath) + + if options.pidFile != "": + f = open(options.pidFile, 'w') + f.write("%s" % os.getpid()) + f.close() + + # httpd-path is specified by standard makefile targets and may be specified + # on the command line to select a particular version of httpd.js. If not + # specified, try to select the one from from the xre bundle, as + # required in bug 882932. + if not options.httpdPath: + options.httpdPath = os.path.join(options.xrePath, "components") + + return options + + +class RemoteArgumentsParser(ReftestArgumentsParser): + def __init__(self, **kwargs): + super(RemoteArgumentsParser, self).__init__() + + # app, xrePath and utilityPath variables are set in main function + self.set_defaults(logFile="reftest.log", + app="", + xrePath="", + utilityPath="", + localLogName=None) + + self.add_argument("--remote-app-path", + action="store", + type=str, + dest="remoteAppPath", + help="Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.") + + self.add_argument("--deviceIP", + action="store", + type=str, + dest="deviceIP", + help="ip address of remote device to test") + + self.add_argument("--deviceSerial", + action="store", + type=str, + dest="deviceSerial", + help="adb serial number of remote device to test") + + self.add_argument("--devicePort", + action="store", + type=str, + default="20701", + dest="devicePort", + help="port of remote device to test") + + self.add_argument("--remote-product-name", + action="store", + type=str, + dest="remoteProductName", + default="fennec", + help="Name of product to test - either fennec or firefox, defaults to fennec") + + self.add_argument("--remote-webserver", + action="store", + type=str, + dest="remoteWebServer", + help="IP Address of the webserver hosting the reftest content") + + self.add_argument("--http-port", + action="store", + type=str, + dest="httpPort", + help="port of the web server for http traffic") + + self.add_argument("--ssl-port", + action="store", + type=str, + dest="sslPort", + help="Port for https traffic to the web server") + + self.add_argument("--remote-logfile", + action="store", + type=str, + dest="remoteLogFile", + default="reftest.log", + help="Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.") + + self.add_argument("--pidfile", + action="store", + type=str, + dest="pidFile", + default="", + help="name of the pidfile to generate") + + self.add_argument("--bootstrap", + action="store_true", + dest="bootstrap", + default=False, + help="test with a bootstrap addon required for native Fennec") + + self.add_argument("--dm_trans", + action="store", + type=str, + dest="dm_trans", + default="sut", + help="the transport to use to communicate with device: [adb|sut]; default=sut") + + self.add_argument("--remoteTestRoot", + action="store", + type=str, + dest="remoteTestRoot", + help="remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)") + + self.add_argument("--httpd-path", + action="store", + type=str, + dest="httpdPath", + help="path to the httpd.js file") + + def validate_remote(self, options, automation): + # Ensure our defaults are set properly for everything we can infer + if not options.remoteTestRoot: + options.remoteTestRoot = automation._devicemanager.deviceRoot + \ + '/reftest' + options.remoteProfile = options.remoteTestRoot + "/profile" + + if options.remoteWebServer is None: + options.remoteWebServer = self.get_ip() + + # Verify that our remotewebserver is set properly + if options.remoteWebServer == '127.0.0.1': + self.error("ERROR: Either you specified the loopback for the remote webserver or ", + "your local IP cannot be detected. Please provide the local ip in --remote-webserver") + + if not options.httpPort: + options.httpPort = automation.DEFAULT_HTTP_PORT + + if not options.sslPort: + options.sslPort = automation.DEFAULT_SSL_PORT + + # One of remoteAppPath (relative path to application) or the app (executable) must be + # set, but not both. If both are set, we destroy the user's selection for app + # so instead of silently destroying a user specificied setting, we + # error. + if options.remoteAppPath and options.app: + self.error( + "ERROR: You cannot specify both the remoteAppPath and the app") + elif options.remoteAppPath: + options.app = options.remoteTestRoot + "/" + options.remoteAppPath + elif options.app is None: + # Neither remoteAppPath nor app are set -- error + self.error("ERROR: You must specify either appPath or app") + + if options.xrePath is None: + self.error( + "ERROR: You must specify the path to the controller xre directory") + else: + # Ensure xrepath is a full path + options.xrePath = os.path.abspath(options.xrePath) + + options.localLogName = options.remoteLogFile + options.remoteLogFile = options.remoteTestRoot + \ + '/' + options.remoteLogFile + + # Ensure that the options.logfile (which the base class uses) is set to + # the remote setting when running remote. Also, if the user set the + # log file name there, use that instead of reusing the remotelogfile as + # above. + if options.logFile: + # If the user specified a local logfile name use that + options.localLogName = options.logFile + + options.logFile = options.remoteLogFile + + if options.pidFile != "": + with open(options.pidFile, 'w') as f: + f.write(str(os.getpid())) + + # httpd-path is specified by standard makefile targets and may be specified + # on the command line to select a particular version of httpd.js. If not + # specified, try to select the one from hostutils.zip, as required in + # bug 882932. + if not options.httpdPath: + options.httpdPath = os.path.join(options.utilityPath, "components") + + if not options.ignoreWindowSize: + parts = automation._devicemanager.getInfo( + 'screen')['screen'][0].split() + width = int(parts[0].split(':')[1]) + height = int(parts[1].split(':')[1]) + if (width < 1050 or height < 1050): + self.error("ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % ( + width, height)) diff --git a/layout/tools/reftest/remotereftest.py b/layout/tools/reftest/remotereftest.py index 697e988d70e..89c1ae3056c 100644 --- a/layout/tools/reftest/remotereftest.py +++ b/layout/tools/reftest/remotereftest.py @@ -9,10 +9,9 @@ import tempfile import traceback # We need to know our current directory so that we can serve our test files from it. -SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))) +SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) -from runreftest import RefTest -from runreftest import ReftestOptions +from runreftest import RefTest, ReftestResolver from automation import Automation import devicemanager import droid @@ -20,159 +19,26 @@ import mozinfo import moznetwork from remoteautomation import RemoteAutomation, fennecLogcatFilters -class RemoteOptions(ReftestOptions): - def __init__(self, automation): - ReftestOptions.__init__(self) - self.automation = automation +import reftestcommandline - defaults = {} - defaults["logFile"] = "reftest.log" - # app, xrePath and utilityPath variables are set in main function - defaults["app"] = "" - defaults["xrePath"] = "" - defaults["utilityPath"] = "" - defaults["runTestsInParallel"] = False - - self.add_option("--remote-app-path", action="store", - type = "string", dest = "remoteAppPath", - help = "Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.") - defaults["remoteAppPath"] = None - - self.add_option("--deviceIP", action="store", - type = "string", dest = "deviceIP", - help = "ip address of remote device to test") - defaults["deviceIP"] = None - - self.add_option("--deviceSerial", action="store", - type = "string", dest = "deviceSerial", - help = "adb serial number of remote device to test") - defaults["deviceSerial"] = None - - self.add_option("--devicePort", action="store", - type = "string", dest = "devicePort", - help = "port of remote device to test") - defaults["devicePort"] = 20701 - - self.add_option("--remote-product-name", action="store", - type = "string", dest = "remoteProductName", - help = "Name of product to test - either fennec or firefox, defaults to fennec") - defaults["remoteProductName"] = "fennec" - - self.add_option("--remote-webserver", action="store", - type = "string", dest = "remoteWebServer", - help = "IP Address of the webserver hosting the reftest content") - defaults["remoteWebServer"] = moznetwork.get_ip() - - self.add_option("--http-port", action = "store", - type = "string", dest = "httpPort", - help = "port of the web server for http traffic") - defaults["httpPort"] = automation.DEFAULT_HTTP_PORT - - self.add_option("--ssl-port", action = "store", - type = "string", dest = "sslPort", - help = "Port for https traffic to the web server") - defaults["sslPort"] = automation.DEFAULT_SSL_PORT - - self.add_option("--remote-logfile", action="store", - type = "string", dest = "remoteLogFile", - help = "Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.") - defaults["remoteLogFile"] = None - - self.add_option("--pidfile", action = "store", - type = "string", dest = "pidFile", - help = "name of the pidfile to generate") - defaults["pidFile"] = "" - - self.add_option("--bootstrap", action="store_true", dest = "bootstrap", - help = "test with a bootstrap addon required for native Fennec") - defaults["bootstrap"] = False - - self.add_option("--dm_trans", action="store", - type = "string", dest = "dm_trans", - help = "the transport to use to communicate with device: [adb|sut]; default=sut") - defaults["dm_trans"] = "sut" - - self.add_option("--remoteTestRoot", action = "store", - type = "string", dest = "remoteTestRoot", - help = "remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)") - defaults["remoteTestRoot"] = None - - self.add_option("--httpd-path", action = "store", - type = "string", dest = "httpdPath", - help = "path to the httpd.js file") - defaults["httpdPath"] = None - - defaults["localLogName"] = None - - self.set_defaults(**defaults) - - def verifyRemoteOptions(self, options): - if options.runTestsInParallel: - self.error("Cannot run parallel tests here") - - # Ensure our defaults are set properly for everything we can infer - if not options.remoteTestRoot: - options.remoteTestRoot = self.automation._devicemanager.deviceRoot + '/reftest' - options.remoteProfile = options.remoteTestRoot + "/profile" - - # Verify that our remotewebserver is set properly - if (options.remoteWebServer == None or - options.remoteWebServer == '127.0.0.1'): - print "ERROR: Either you specified the loopback for the remote webserver or ", - print "your local IP cannot be detected. Please provide the local ip in --remote-webserver" - return None - - # One of remoteAppPath (relative path to application) or the app (executable) must be - # set, but not both. If both are set, we destroy the user's selection for app - # so instead of silently destroying a user specificied setting, we error. - if (options.remoteAppPath and options.app): - print "ERROR: You cannot specify both the remoteAppPath and the app" - return None - elif (options.remoteAppPath): - options.app = options.remoteTestRoot + "/" + options.remoteAppPath - elif (options.app == None): - # Neither remoteAppPath nor app are set -- error - print "ERROR: You must specify either appPath or app" - return None - - if (options.xrePath == None): - print "ERROR: You must specify the path to the controller xre directory" - return None +class RemoteReftestResolver(ReftestResolver): + def absManifestPath(self, path): + script_abs_path = os.path.join(SCRIPT_DIRECTORY, path) + if os.path.exists(script_abs_path): + rv = script_abs_path + elif os.path.exists(os.path.abspath(path)): + rv = os.path.abspath(path) else: - # Ensure xrepath is a full path - options.xrePath = os.path.abspath(options.xrePath) + print >> sys.stderr, "Could not find manifest %s" % script_abs_path + sys.exit(1) + return os.path.normpath(rv) - # Default to /reftest/reftest.log - if (options.remoteLogFile == None): - options.remoteLogFile = 'reftest.log' + def manifestURL(self, options, path): + # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot + # It's possible for this url to have a leading "..", but reftest.js will fix that up + relPath = os.path.relpath(path, SCRIPT_DIRECTORY) + return "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, relPath) - options.localLogName = options.remoteLogFile - options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile - - # Ensure that the options.logfile (which the base class uses) is set to - # the remote setting when running remote. Also, if the user set the - # log file name there, use that instead of reusing the remotelogfile as above. - if (options.logFile): - # If the user specified a local logfile name use that - options.localLogName = options.logFile - - options.logFile = options.remoteLogFile - - if (options.pidFile != ""): - f = open(options.pidFile, 'w') - f.write("%s" % os.getpid()) - f.close() - - # httpd-path is specified by standard makefile targets and may be specified - # on the command line to select a particular version of httpd.js. If not - # specified, try to select the one from hostutils.zip, as required in bug 882932. - if not options.httpdPath: - options.httpdPath = os.path.join(options.utilityPath, "components") - - # TODO: Copied from main, but I think these are no longer used in a post xulrunner world - #options.xrePath = options.remoteTestRoot + self.automation._product + '/xulrunner' - #options.utilityPath = options.testRoot + self.automation._product + '/bin' - return options class ReftestServer: """ Web server used to serve Reftests, for closer fidelity to the real web. @@ -260,6 +126,7 @@ class ReftestServer: class RemoteReftest(RefTest): remoteApp = '' + resolver_cls = RemoteReftestResolver def __init__(self, automation, devicemanager, options, scriptDir): RefTest.__init__(self) @@ -342,8 +209,12 @@ class RemoteReftest(RefTest): def stopWebServer(self, options): self.server.stop() - def createReftestProfile(self, options, reftestlist): - profile = RefTest.createReftestProfile(self, options, reftestlist, server=options.remoteWebServer, port=options.httpPort) + def createReftestProfile(self, options, manifest): + profile = RefTest.createReftestProfile(self, + options, + manifest, + server=options.remoteWebServer, + port=options.httpPort) profileDir = profile.profile prefs = {} @@ -355,7 +226,6 @@ class RemoteReftest(RefTest): # Set a future policy version to avoid the telemetry prompt. prefs["toolkit.telemetry.prompted"] = 999 prefs["toolkit.telemetry.notifiedOptOut"] = 999 - prefs["reftest.uri"] = "%s" % reftestlist prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True # Point the url-classifier to the local testing server for fast failures @@ -410,9 +280,6 @@ class RemoteReftest(RefTest): print "Automation Error: Failed to copy extra files to device" raise - def getManifestPath(self, path): - return path - def printDeviceInfo(self, printLogcat=False): try: if printLogcat: @@ -473,10 +340,10 @@ class RemoteReftest(RefTest): except: print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % self.pidFile -def main(args): +def main(): automation = RemoteAutomation(None) - parser = RemoteOptions(automation) - options, args = parser.parse_args() + parser = reftestcommandline.RemoteArgumentsParser() + options = parser.parse_args() if (options.dm_trans == 'sut' and options.deviceIP == None): print "Error: If --dm_trans = sut, you must provide a device IP to connect to via the --deviceIP option" @@ -502,18 +369,7 @@ def main(args): automation.setProduct(options.remoteProductName) # Set up the defaults and ensure options are set - options = parser.verifyRemoteOptions(options) - if (options == None): - print "ERROR: Invalid options specified, use --help for a list of valid options" - return 1 - - if not options.ignoreWindowSize: - parts = dm.getInfo('screen')['screen'][0].split() - width = int(parts[0].split(':')[1]) - height = int(parts[1].split(':')[1]) - if (width < 1050 or height < 1050): - print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (width, height) - return 1 + parser.validate_remote(options, automation) # Check that Firefox is installed expected = options.app.split('/')[-1] @@ -526,7 +382,7 @@ def main(args): automation.setRemoteProfile(options.remoteProfile) automation.setRemoteLog(options.remoteLogFile) reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY) - options = parser.verifyCommonOptions(options, reftest) + parser.validate(options, reftest) if mozinfo.info['debug']: print "changing timeout for remote debug reftests from %s to 600 seconds" % options.timeout @@ -535,17 +391,6 @@ def main(args): # Hack in a symbolic link for jsreftest os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest"))) - # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot - manifest = args[0] - if os.path.exists(os.path.join(SCRIPT_DIRECTORY, args[0])): - manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + args[0] - elif os.path.exists(args[0]): - manifestPath = os.path.abspath(args[0]).split(SCRIPT_DIRECTORY)[1].strip('/') - manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + manifestPath - else: - print "ERROR: Could not find test manifest '%s'" % manifest - return 1 - # Start the webserver retVal = reftest.startWebServer(options) if retVal: @@ -561,11 +406,8 @@ def main(args): # manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list" retVal = 0 try: - cmdlineArgs = ["-reftest", manifest] - if options.bootstrap: - cmdlineArgs = [] dm.recordLogcat() - retVal = reftest.runTests(manifest, options, cmdlineArgs) + retVal = reftest.runTests(options.tests, options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() @@ -578,5 +420,5 @@ def main(args): return retVal if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) + sys.exit(main()) diff --git a/layout/tools/reftest/runreftest.py b/layout/tools/reftest/runreftest.py index 32d18f68ba1..353a745c4d0 100644 --- a/layout/tools/reftest/runreftest.py +++ b/layout/tools/reftest/runreftest.py @@ -6,9 +6,8 @@ Runs the reftest test harness. """ -from optparse import OptionParser -from urlparse import urlparse import collections +import json import multiprocessing import os import re @@ -19,8 +18,9 @@ import sys import threading SCRIPT_DIRECTORY = os.path.abspath( - os.path.realpath(os.path.dirname(sys.argv[0]))) -sys.path.insert(0, SCRIPT_DIRECTORY) + os.path.realpath(os.path.dirname(__file__))) +if SCRIPT_DIRECTORY not in sys.path: + sys.path.insert(0, SCRIPT_DIRECTORY) import mozcrash import mozdebug @@ -32,13 +32,7 @@ import mozrunner from mozrunner.utils import get_stack_fixer_function, test_environment from mozscreenshot import printstatus, dump_screen -here = os.path.abspath(os.path.dirname(__file__)) - -try: - from mozbuild.base import MozbuildObject - build_obj = MozbuildObject.from_environment(cwd=here) -except ImportError: - build_obj = None +import reftestcommandline # set up logging handler a la automation.py.in for compatability import logging @@ -132,14 +126,63 @@ class ReftestThread(threading.Thread): if summaryHeadRegex.search(line) is None: yield line +class ReftestResolver(object): + def defaultManifest(self, suite): + return {"reftest": "reftest.list", + "crashtest": "crashtests.list", + "jstestbrowser": "jstests.list"}[suite] + + def findManifest(self, suite, test_file): + """Return a tuple of (manifest-path, filter-string) for running test_file. + + test_file is a path to a test or a manifest file + """ + if not os.path.isabs(test_file): + test_file = self.absManifestPath(test_file) + + if os.path.isdir(test_file): + return os.path.join(test_file, self.defaultManifest(suite)), None + + if test_file.endswith('.list'): + return test_file, None + + return (self.findManifest(suite, os.path.dirname(test_file))[0], + r".*(?:/|\\)%s$" % os.path.basename(test_file)) + + def absManifestPath(self, path): + return os.path.abspath(path) + + def manifestURL(self, options, path): + return "file://%s" % path + + def resolveManifests(self, options, tests): + suite = options.suite + manifests = {} + for testPath in tests: + manifest, filter_str = self.findManifest(suite, testPath) + manifest = self.manifestURL(options, manifest) + if manifest not in manifests: + manifests[manifest] = set() + if filter_str is not None: + manifests[manifest].add(filter_str) + + for key in manifests.iterkeys(): + if os.path.split(key)[1] != self.defaultManifest(suite): + print >> sys.stderr, "Invalid manifest for suite %s, %s" %(options.suite, key) + sys.exit(1) + manifests[key] = sorted(list(manifests[key])) + + return manifests class RefTest(object): oldcwd = os.getcwd() + resolver_cls = ReftestResolver def __init__(self): self.update_mozinfo() self.lastTestSeen = 'reftest' self.haveDumpedScreen = False + self.resolver = self.resolver_cls() def update_mozinfo(self): """walk up directories to find mozinfo.json update the info""" @@ -158,30 +201,15 @@ class RefTest(object): "Get an absolute path relative to self.oldcwd." return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path))) - def getManifestPath(self, path): - "Get the path of the manifest, and for remote testing this function is subclassed to point to remote manifest" - path = self.getFullPath(path) - if os.path.isdir(path): - defaultManifestPath = os.path.join(path, 'reftest.list') - if os.path.exists(defaultManifestPath): - path = defaultManifestPath - else: - defaultManifestPath = os.path.join(path, 'crashtests.list') - if os.path.exists(defaultManifestPath): - path = defaultManifestPath - return path + def createReftestProfile(self, options, manifests, server='localhost', port=0, + profile_to_clone=None): + """Sets up a profile for reftest. - def makeJSString(self, s): - return '"%s"' % re.sub(r'([\\"])', r'\\\1', s) - - def createReftestProfile(self, options, manifest, server='localhost', port=0, - special_powers=True, profile_to_clone=None): - """ - Sets up a profile for reftest. - 'manifest' is the path to the reftest.list file we want to test with. This is used in - the remote subclass in remotereftest.py so we can write it to a preference for the - bootstrap extension. - """ + :param options: Object containing command line options + :param manifests: Dictionary of the form {manifest_path: [filters]} + :param server: Server name to use for http tests + :param profile_to_clone: Path to a profile to use as the basis for the + test profile""" locations = mozprofile.permissions.ServerLocations() locations.add_host(server, scheme='http', port=port) @@ -200,12 +228,10 @@ class RefTest(object): prefs['reftest.logFile'] = options.logFile if options.ignoreWindowSize: prefs['reftest.ignoreWindowSize'] = True - if options.filter: - prefs['reftest.filter'] = options.filter if options.shuffle: prefs['reftest.shuffle'] = True prefs['reftest.focusFilterMode'] = options.focusFilterMode - prefs['reftest.uri'] = "file://%s" % os.path.abspath(manifest) + prefs['reftest.manifests'] = json.dumps(manifests) # Ensure that telemetry is disabled, so we don't connect to the telemetry # server in the middle of the tests. @@ -250,13 +276,10 @@ class RefTest(object): thispref[1].strip()) # install the reftest extension bits into the profile - addons = [] - addons.append(os.path.join(SCRIPT_DIRECTORY, "reftest")) + addons = [options.reftestExtensionPath] - # I would prefer to use "--install-extension reftest/specialpowers", but that requires tight coordination with - # release engineering and landing on multiple branches at once. - if special_powers and (manifest.endswith('crashtests.list') or manifest.endswith('jstests.list')): - addons.append(os.path.join(SCRIPT_DIRECTORY, 'specialpowers')) + if options.specialPowersExtensionPath is not None: + addons.append(options.specialPowersExtensionPath) # SpecialPowers requires insecure automation-only features that we # put behind a pref. prefs['security.turn_off_all_security_so_that_viruses_can_take_over_this_computer'] = True @@ -337,7 +360,7 @@ class RefTest(object): if profileDir: shutil.rmtree(profileDir, True) - def runTests(self, testPath, options, cmdlineArgs=None): + def runTests(self, tests, options, cmdlineArgs=None): # Despite our efforts to clean up servers started by this script, in practice # we still see infrequent cases where a process is orphaned and interferes # with future tests, typically because the old server is keeping the port in use. @@ -346,8 +369,12 @@ class RefTest(object): self.killNamedOrphans('ssltunnel') self.killNamedOrphans('xpcshell') - if not options.runTestsInParallel: - return self.runSerialTests(testPath, options, cmdlineArgs) + manifests = self.resolver.resolveManifests(options, tests) + if options.filter: + manifests[""] = options.filter + + if not hasattr(options, "runTestsInParallel") or not options.runTestsInParallel: + return self.runSerialTests(manifests, options, cmdlineArgs) cpuCount = multiprocessing.cpu_count() @@ -378,7 +405,6 @@ class RefTest(object): jobArgs.remove("--run-tests-in-parallel") except: pass - jobArgs.insert(-1, "--no-run-tests-in-parallel") jobArgs[0:0] = [sys.executable, "-u"] threads = [ReftestThread(args) for args in perProcessArgs[1:]] @@ -601,7 +627,7 @@ class RefTest(object): status = 1 return status - def runSerialTests(self, testPath, options, cmdlineArgs=None): + def runSerialTests(self, manifests, options, cmdlineArgs=None): debuggerInfo = None if options.debugger: debuggerInfo = mozdebug.get_debugger_info(options.debugger, options.debuggerArgs, @@ -609,10 +635,9 @@ class RefTest(object): profileDir = None try: - reftestlist = self.getManifestPath(testPath) if cmdlineArgs == None: cmdlineArgs = [] - profile = self.createReftestProfile(options, reftestlist) + profile = self.createReftestProfile(options, manifests) profileDir = profile.profile # name makes more sense # browser environment @@ -660,209 +685,26 @@ class RefTest(object): continue -class ReftestOptions(OptionParser): - - def __init__(self): - OptionParser.__init__(self) - defaults = {} - self.add_option("--xre-path", - action="store", type="string", dest="xrePath", - # individual scripts will set a sane default - default=None, - help="absolute path to directory containing XRE (probably xulrunner)") - self.add_option("--symbols-path", - action="store", type="string", dest="symbolsPath", - default=None, - help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols") - self.add_option("--debugger", - action="store", dest="debugger", - help="use the given debugger to launch the application") - self.add_option("--debugger-args", - action="store", dest="debuggerArgs", - help="pass the given args to the debugger _before_ " - "the application on the command line") - self.add_option("--debugger-interactive", - action="store_true", dest="debuggerInteractive", - help="prevents the test harness from redirecting " - "stdout and stderr for interactive debuggers") - self.add_option("--appname", - action="store", type="string", dest="app", - help="absolute path to application, overriding default") - # Certain paths do not make sense when we're cross compiling Fennec. This - # logic is cribbed from the example in - # python/mozbuild/mozbuild/mach_commands.py. - defaults['app'] = build_obj.get_binary_path() if \ - build_obj and build_obj.substs['MOZ_BUILD_APP'] != 'mobile/android' else None - - self.add_option("--extra-profile-file", - action="append", dest="extraProfileFiles", - default=[], - help="copy specified files/dirs to testing profile") - self.add_option("--timeout", - action="store", dest="timeout", type="int", - default=5 * 60, # 5 minutes per bug 479518 - help="reftest will timeout in specified number of seconds. [default %default s].") - self.add_option("--leak-threshold", - action="store", type="int", dest="defaultLeakThreshold", - default=0, - help="fail if the number of bytes leaked in default " - "processes through refcounted objects (or bytes " - "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) " - "is greater than the given number") - self.add_option("--utility-path", - action="store", type="string", dest="utilityPath", - help="absolute path to directory containing utility " - "programs (xpcshell, ssltunnel, certutil)") - defaults["utilityPath"] = build_obj.bindir if \ - build_obj and build_obj.substs['MOZ_BUILD_APP'] != 'mobile/android' else None - - self.add_option("--total-chunks", - type="int", dest="totalChunks", - help="how many chunks to split the tests up into") - defaults["totalChunks"] = None - - self.add_option("--this-chunk", - type="int", dest="thisChunk", - help="which chunk to run between 1 and --total-chunks") - defaults["thisChunk"] = None - - self.add_option("--log-file", - action="store", type="string", dest="logFile", - default=None, - help="file to log output to in addition to stdout") - defaults["logFile"] = None - - self.add_option("--skip-slow-tests", - dest="skipSlowTests", action="store_true", - help="skip tests marked as slow when running") - defaults["skipSlowTests"] = False - - self.add_option("--ignore-window-size", - dest="ignoreWindowSize", action="store_true", - help="ignore the window size, which may cause spurious failures and passes") - defaults["ignoreWindowSize"] = False - - self.add_option("--install-extension", - action="append", dest="extensionsToInstall", - help="install the specified extension in the testing profile. " - "The extension file's name should be .xpi where is " - "the extension's id as indicated in its install.rdf. " - "An optional path can be specified too.") - defaults["extensionsToInstall"] = [] - - self.add_option("--run-tests-in-parallel", - action="store_true", dest="runTestsInParallel", - help="run tests in parallel if possible") - self.add_option("--no-run-tests-in-parallel", - action="store_false", dest="runTestsInParallel", - help="do not run tests in parallel") - defaults["runTestsInParallel"] = False - - self.add_option("--setenv", - action="append", type="string", - dest="environment", metavar="NAME=VALUE", - help="sets the given variable in the application's " - "environment") - defaults["environment"] = [] - - self.add_option("--filter", - action="store", type="string", dest="filter", - help="specifies a regular expression (as could be passed to the JS " - "RegExp constructor) to test against URLs in the reftest manifest; " - "only test items that have a matching test URL will be run.") - defaults["filter"] = None - - self.add_option("--shuffle", - action="store_true", dest="shuffle", - help="run reftests in random order") - defaults["shuffle"] = False - - self.add_option("--focus-filter-mode", - action="store", type="string", dest="focusFilterMode", - help="filters tests to run by whether they require focus. " - "Valid values are `all', `needs-focus', or `non-needs-focus'. " - "Defaults to `all'.") - defaults["focusFilterMode"] = "all" - - self.add_option("--e10s", - action="store_true", - dest="e10s", - help="enables content processes") - defaults["e10s"] = False - - self.add_option("--setpref", - action="append", type="string", - default=[], - dest="extraPrefs", metavar="PREF=VALUE", - help="defines an extra user preference") - - self.set_defaults(**defaults) - - def verifyCommonOptions(self, options, reftest): - if options.totalChunks is not None and options.thisChunk is None: - self.error("thisChunk must be specified when totalChunks is specified") - - if options.totalChunks: - if not 1 <= options.thisChunk <= options.totalChunks: - self.error("thisChunk must be between 1 and totalChunks") - - if options.logFile: - options.logFile = reftest.getFullPath(options.logFile) - - if options.xrePath is not None: - if not os.access(options.xrePath, os.F_OK): - self.error("--xre-path '%s' not found" % options.xrePath) - if not os.path.isdir(options.xrePath): - self.error("--xre-path '%s' is not a directory" % - options.xrePath) - options.xrePath = reftest.getFullPath(options.xrePath) - - if options.runTestsInParallel: - if options.logFile is not None: - self.error("cannot specify logfile with parallel tests") - if options.totalChunks is not None and options.thisChunk is None: - self.error("cannot specify thisChunk or totalChunks with parallel tests") - if options.focusFilterMode != "all": - self.error("cannot specify focusFilterMode with parallel tests") - if options.debugger is not None: - self.error("cannot specify a debugger with parallel tests") - - options.leakThresholds = { - "default": options.defaultLeakThreshold, - "tab": 5000, # See dependencies of bug 1051230. - } - - return options +def run(**kwargs): + # Mach gives us kwargs; this is a way to turn them back into an + # options object + parser = reftestcommandline.DesktopArgumentsParser() + reftest = RefTest() + parser.set_defaults(**kwargs) + options = parser.parse_args(kwargs["tests"]) + parser.validate(options, reftest) + return reftest.runTests(options.tests, options) def main(): - parser = ReftestOptions() + parser = reftestcommandline.DesktopArgumentsParser() reftest = RefTest() - options, args = parser.parse_args() - if len(args) != 1: - print >>sys.stderr, "No reftest.list specified." - sys.exit(1) + options = parser.parse_args() + parser.validate(options, reftest) - options = parser.verifyCommonOptions(options, reftest) - if options.app is None: - parser.error("could not find the application path, --appname must be specified") + sys.exit(reftest.runTests(options.tests, options)) - options.app = reftest.getFullPath(options.app) - if not os.path.exists(options.app): - print """Error: Path %(app)s doesn't exist. -Are you executing $objdir/_tests/reftest/runreftest.py?""" \ - % {"app": options.app} - sys.exit(1) - - if options.xrePath is None: - options.xrePath = os.path.dirname(options.app) - - if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2: - options.symbolsPath = reftest.getFullPath(options.symbolsPath) - options.utilityPath = reftest.getFullPath(options.utilityPath) - - sys.exit(reftest.runTests(args[0], options)) if __name__ == "__main__": main() diff --git a/layout/tools/reftest/runreftestb2g.py b/layout/tools/reftest/runreftestb2g.py index 085a246ac00..c0a9257e93e 100644 --- a/layout/tools/reftest/runreftestb2g.py +++ b/layout/tools/reftest/runreftestb2g.py @@ -10,207 +10,18 @@ import traceback # We need to know our current directory so that we can serve our test files from it. here = os.path.abspath(os.path.dirname(__file__)) +if here not in sys.path: + sys.path.insert(0, here) from automation import Automation from b2gautomation import B2GRemoteAutomation from b2g_desktop import run_desktop_reftests +from remotereftest import RemoteReftestResolver, ReftestServer from runreftest import RefTest -from runreftest import ReftestOptions -from remotereftest import ReftestServer +import reftestcommandline from mozdevice import DeviceManagerADB, DMError from marionette import Marionette -import moznetwork - -class B2GOptions(ReftestOptions): - - def __init__(self, **kwargs): - defaults = {} - ReftestOptions.__init__(self) - # This is only used for procName in run_remote_reftests. - defaults["app"] = Automation.DEFAULT_APP - - self.add_option("--browser-arg", action="store", - type = "string", dest = "browser_arg", - help = "Optional command-line arg to pass to the browser") - defaults["browser_arg"] = None - - self.add_option("--b2gpath", action="store", - type = "string", dest = "b2gPath", - help = "path to B2G repo or qemu dir") - defaults["b2gPath"] = None - - self.add_option("--marionette", action="store", - type = "string", dest = "marionette", - help = "host:port to use when connecting to Marionette") - defaults["marionette"] = None - - self.add_option("--emulator", action="store", - type="string", dest = "emulator", - help = "Architecture of emulator to use: x86 or arm") - defaults["emulator"] = None - self.add_option("--emulator-res", action="store", - type="string", dest = "emulator_res", - help = "Emulator resolution of the format 'x'") - defaults["emulator_res"] = None - - self.add_option("--no-window", action="store_true", - dest = "noWindow", - help = "Pass --no-window to the emulator") - defaults["noWindow"] = False - - self.add_option("--adbpath", action="store", - type = "string", dest = "adb_path", - help = "path to adb") - defaults["adb_path"] = "adb" - - self.add_option("--deviceIP", action="store", - type = "string", dest = "deviceIP", - help = "ip address of remote device to test") - defaults["deviceIP"] = None - - self.add_option("--devicePort", action="store", - type = "string", dest = "devicePort", - help = "port of remote device to test") - defaults["devicePort"] = 20701 - - self.add_option("--remote-logfile", action="store", - type = "string", dest = "remoteLogFile", - help = "Name of log file on the device relative to the device root. PLEASE ONLY USE A FILENAME.") - defaults["remoteLogFile"] = None - - self.add_option("--remote-webserver", action = "store", - type = "string", dest = "remoteWebServer", - help = "ip address where the remote web server is hosted at") - defaults["remoteWebServer"] = None - - self.add_option("--http-port", action = "store", - type = "string", dest = "httpPort", - help = "ip address where the remote web server is hosted at") - defaults["httpPort"] = None - - self.add_option("--ssl-port", action = "store", - type = "string", dest = "sslPort", - help = "ip address where the remote web server is hosted at") - defaults["sslPort"] = None - - self.add_option("--pidfile", action = "store", - type = "string", dest = "pidFile", - help = "name of the pidfile to generate") - defaults["pidFile"] = "" - self.add_option("--gecko-path", action="store", - type="string", dest="geckoPath", - help="the path to a gecko distribution that should " - "be installed on the emulator prior to test") - defaults["geckoPath"] = None - self.add_option("--logdir", action="store", - type="string", dest="logdir", - help="directory to store log files") - defaults["logdir"] = None - self.add_option('--busybox', action='store', - type='string', dest='busybox', - help="Path to busybox binary to install on device") - defaults['busybox'] = None - self.add_option("--httpd-path", action = "store", - type = "string", dest = "httpdPath", - help = "path to the httpd.js file") - defaults["httpdPath"] = None - self.add_option("--profile", action="store", - type="string", dest="profile", - help="for desktop testing, the path to the " - "gaia profile to use") - defaults["profile"] = None - self.add_option("--desktop", action="store_true", - dest="desktop", - help="Run the tests on a B2G desktop build") - defaults["desktop"] = False - self.add_option("--mulet", action="store_true", - dest="mulet", - help="Run the tests on a B2G desktop build") - defaults["mulet"] = False - self.add_option("--enable-oop", action="store_true", - dest="oop", - help="Run the tests out of process") - defaults["oop"] = False - defaults["remoteTestRoot"] = None - defaults["logFile"] = "reftest.log" - defaults["autorun"] = True - defaults["closeWhenDone"] = True - defaults["testPath"] = "" - defaults["runTestsInParallel"] = False - - self.set_defaults(**defaults) - - def verifyRemoteOptions(self, options, auto): - if options.runTestsInParallel: - self.error("Cannot run parallel tests here") - - if not options.remoteTestRoot: - options.remoteTestRoot = auto._devicemanager.deviceRoot + "/reftest" - - options.remoteProfile = options.remoteTestRoot + "/profile" - - productRoot = options.remoteTestRoot + "/" + auto._product - if options.utilityPath is None: - options.utilityPath = productRoot + "/bin" - - if options.remoteWebServer == None: - if os.name != "nt": - options.remoteWebServer = moznetwork.get_ip() - else: - print "ERROR: you must specify a --remote-webserver=\n" - return None - - options.webServer = options.remoteWebServer - - if not options.httpPort: - options.httpPort = auto.DEFAULT_HTTP_PORT - - if not options.sslPort: - options.sslPort = auto.DEFAULT_SSL_PORT - - if options.geckoPath and not options.emulator: - self.error("You must specify --emulator if you specify --gecko-path") - - if options.logdir and not options.emulator: - self.error("You must specify --emulator if you specify --logdir") - - #if not options.emulator and not options.deviceIP: - # print "ERROR: you must provide a device IP" - # return None - - if options.remoteLogFile == None: - options.remoteLogFile = "reftest.log" - - options.localLogName = options.remoteLogFile - options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile - - # Ensure that the options.logfile (which the base class uses) is set to - # the remote setting when running remote. Also, if the user set the - # log file name there, use that instead of reusing the remotelogfile as above. - if (options.logFile): - # If the user specified a local logfile name use that - options.localLogName = options.logFile - options.logFile = options.remoteLogFile - - # Only reset the xrePath if it wasn't provided - if options.xrePath == None: - options.xrePath = options.utilityPath - options.xrePath = os.path.abspath(options.xrePath) - - if options.pidFile != "": - f = open(options.pidFile, 'w') - f.write("%s" % os.getpid()) - f.close() - - # httpd-path is specified by standard makefile targets and may be specified - # on the command line to select a particular version of httpd.js. If not - # specified, try to select the one from from the xre bundle, as required in bug 882932. - if not options.httpdPath: - options.httpdPath = os.path.join(options.xrePath, "components") - - return options - class ProfileConfigParser(ConfigParser.RawConfigParser): """Subclass of RawConfigParser that outputs .ini files in the exact @@ -243,6 +54,7 @@ class B2GRemoteReftest(RefTest): localProfile = None remoteApp = '' profile = None + resolver_cls = RemoteReftestResolver def __init__(self, automation, devicemanager, options, scriptDir): RefTest.__init__(self) @@ -418,10 +230,9 @@ class B2GRemoteReftest(RefTest): pass - def createReftestProfile(self, options, reftestlist): - profile = RefTest.createReftestProfile(self, options, reftestlist, - server=options.remoteWebServer, - special_powers=False) + def createReftestProfile(self, options, manifests): + profile = RefTest.createReftestProfile(self, options, manifests, + server=options.remoteWebServer) profileDir = profile.profile prefs = {} @@ -437,7 +248,7 @@ class B2GRemoteReftest(RefTest): prefs["network.dns.localDomains"] = "app://test-container.gaiamobile.org" prefs["reftest.browser.iframe.enabled"] = False prefs["reftest.remote"] = True - prefs["reftest.uri"] = "%s" % reftestlist + # Set a future policy version to avoid the telemetry prompt. prefs["toolkit.telemetry.prompted"] = 999 prefs["toolkit.telemetry.notifiedOptOut"] = 999 @@ -495,9 +306,6 @@ class B2GRemoteReftest(RefTest): print "Automation Error: Failed to copy extra files to device" raise - def getManifestPath(self, path): - return path - def environment(self, **kwargs): return self.automation.environment(**kwargs) @@ -516,7 +324,7 @@ class B2GRemoteReftest(RefTest): return status -def run_remote_reftests(parser, options, args): +def run_remote_reftests(parser, options): auto = B2GRemoteAutomation(None, "fennec", context_chrome=True) # create our Marionette instance @@ -559,11 +367,7 @@ def run_remote_reftests(parser, options, args): dm = DeviceManagerADB(**kwargs) auto.setDeviceManager(dm) - options = parser.verifyRemoteOptions(options, auto) - - if (options == None): - print "ERROR: Invalid options specified, use --help for a list of valid options" - sys.exit(1) + parser.validate_remote(options, auto) # TODO fix exception if not options.ignoreWindowSize: @@ -580,7 +384,7 @@ def run_remote_reftests(parser, options, args): auto.logFinish = "REFTEST TEST-START | Shutdown" reftest = B2GRemoteReftest(auto, dm, options, here) - options = parser.verifyCommonOptions(options, reftest) + parser.validate(options, reftest) logParent = os.path.dirname(options.remoteLogFile) dm.mkDir(logParent); @@ -590,16 +394,6 @@ def run_remote_reftests(parser, options, args): # Hack in a symbolic link for jsreftest os.system("ln -s %s %s" % (os.path.join('..', 'jsreftest'), os.path.join(here, 'jsreftest'))) - # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot - manifest = args[0] - if os.path.exists(os.path.join(here, args[0])): - manifest = "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, args[0]) - elif os.path.exists(args[0]): - manifestPath = os.path.abspath(args[0]).split(here)[1].strip('/') - manifest = "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, manifestPath) - else: - print "ERROR: Could not find test manifest '%s'" % manifest - return 1 # Start the webserver retVal = 1 @@ -611,11 +405,7 @@ def run_remote_reftests(parser, options, args): if (dm.processExist(procName)): dm.killProcess(procName) - cmdlineArgs = ["-reftest", manifest] - if getattr(options, 'bootstrap', False): - cmdlineArgs = [] - - retVal = reftest.runTests(manifest, options, cmdlineArgs) + retVal = reftest.runTests(options.tests, options) except: print "Automation Error: Exception caught while running tests" traceback.print_exc() @@ -629,13 +419,21 @@ def run_remote_reftests(parser, options, args): reftest.stopWebServer(options) return retVal -def main(args=sys.argv[1:]): - parser = B2GOptions() - options, args = parser.parse_args(args) +def run_remote(**kwargs): + # Tests need to be served from a subdirectory of the server. Symlink + # topsrcdir here to get around this. + parser = reftestcommandline.B2GArgumentParser() + parser.set_defaults(**kwargs) + options = parser.parse_args(kwargs["tests"]) + return run_remote_reftests(parser, options) + +def main(): + parser = reftestcommandline.B2GArgumentParser() + options = parser.parse_args() if options.desktop or options.mulet: - return run_desktop_reftests(parser, options, args) - return run_remote_reftests(parser, options, args) + return run_desktop_reftests(parser, options) + return run_remote_reftests(parser, options) if __name__ == "__main__": diff --git a/testing/testsuite-targets.mk b/testing/testsuite-targets.mk index 1e99d205691..98c7b76e8a0 100644 --- a/testing/testsuite-targets.mk +++ b/testing/testsuite-targets.mk @@ -174,7 +174,7 @@ REMOTE_REFTEST = rm -f ./$@.log && $(PYTHON) _tests/reftest/remotereftest.py \ --dm_trans=$(DM_TRANS) --ignore-window-size \ --app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \ --httpd-path=_tests/modules \ - $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) '$(1)' | tee ./$@.log + $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(1) | tee ./$@.log RUN_REFTEST_B2G = rm -f ./$@.log && $(PYTHON) _tests/reftest/runreftestb2g.py \ --remote-webserver=10.0.2.2 --b2gpath=${B2G_PATH} --adbpath=${ADB_PATH} \ @@ -209,7 +209,7 @@ reftest-remote: echo 'please prepare your host with the environment variable TEST_DEVICE'; \ else \ ln -s $(abspath $(topsrcdir)) _tests/reftest/tests; \ - $(call REMOTE_REFTEST,tests/$(TEST_PATH)); \ + $(call REMOTE_REFTEST,'tests/$(TEST_PATH)'); \ $(CHECK_TEST_ERROR); \ fi