wine/gecko
0
0
Fork 0
mirror of https://gitlab.winehq.org/wine/wine-gecko.git synced 2024-09-13 09:24:08 -07:00
Code Issues Packages Projects Releases Wiki Activity

Bug 1181516 - Allow reftests to take paths to multiple directories containing tests on the command line, r=jmaher

Browse Source 
This makes reftest command line arguments behave more like other test suites,
so we can use a simple unified syntax for e.g. |mach try|. The patch also
reworks the command line argument parsing to use argparse rather than optparse,
and causes mach to reuse the same parser as the suite.
This commit is contained in:
James Graham 2015-08-25 14:07:23 +01:00
parent 38ba879b87
commit af13c2bae8
9 changed files with 1077 additions and 1034 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
layout/tools/reftest/Makefile.in
View File

@ -7,6 +7,7 @@ _DEST_DIR = $(DEPTH)/_tests/reftest
_HARNESS_FILES = \
$(srcdir)/runreftest.py \
$(srcdir)/reftestcommandline.py \
$(srcdir)/remotereftest.py \
$(srcdir)/runreftestb2g.py \
$(srcdir)/b2g_desktop.py \

23
layout/tools/reftest/b2g_desktop.py
View File

@ -9,7 +9,7 @@ import sys
here = os.path.abspath(os.path.dirname(__file__))
from runreftest import RefTest, ReftestOptions
from runreftest import RefTest
from marionette_driver import expected
from marionette_driver.by import By
@ -51,12 +51,10 @@ class B2GDesktopReftest(RefTest):
f.close()
self.marionette.execute_script(self.test_script)
def run_tests(self, test_path, options):
reftestlist = self.getManifestPath(test_path)
if not reftestlist.startswith('file://'):
reftestlist = 'file://%s' % reftestlist
def run_tests(self, tests, options):
manifests = self.resolver.resolveManifests(options, tests)
self.profile = self.create_profile(options, reftestlist,
self.profile = self.create_profile(options, manifests,
profile_to_clone=options.profile)
env = self.buildBrowserEnv(options, self.profile.profile)
kp_kwargs = { 'processOutputLine': [self._on_output],
@ -107,8 +105,8 @@ class B2GDesktopReftest(RefTest):
log.info("%s | Running tests: end.", os.path.basename(__file__))
return status
def create_profile(self, options, reftestlist, profile_to_clone=None):
profile = RefTest.createReftestProfile(self, options, reftestlist,
def create_profile(self, options, manifests, profile_to_clone=None):
profile = RefTest.createReftestProfile(self, options, manifests,
profile_to_clone=profile_to_clone)
prefs = {}
@ -136,7 +134,6 @@ class B2GDesktopReftest(RefTest):
prefs["network.dns.localDomains"] = "app://test-container.gaiamobile.org"
prefs["reftest.browser.iframe.enabled"] = False
prefs["reftest.remote"] = False
prefs["reftest.uri"] = "%s" % reftestlist
# Set a future policy version to avoid the telemetry prompt.
prefs["toolkit.telemetry.prompted"] = 999
prefs["toolkit.telemetry.notifiedOptOut"] = 999
@ -192,7 +189,7 @@ class MuletReftest(B2GDesktopReftest):
Wait(self.marionette, timeout).until(expected.element_present(
By.CSS_SELECTOR, '#homescreen[loading-state=false]'))
def run_desktop_reftests(parser, options, args):
def run_desktop_reftests(parser, options):
marionette_args = {}
if options.marionette:
host, port = options.marionette.split(':')
@ -204,9 +201,7 @@ def run_desktop_reftests(parser, options, args):
else:
reftest = B2GDesktopReftest(marionette_args)
options = ReftestOptions.verifyCommonOptions(parser, options, reftest)
if options == None:
sys.exit(1)
parser.validate(options, reftest)
# add a -bin suffix if b2g-bin exists, but just b2g was specified
if options.app[-4:] != '-bin':
@ -219,4 +214,4 @@ def run_desktop_reftests(parser, options, args):
if options.desktop and not options.profile:
raise Exception("must specify --profile when specifying --desktop")
sys.exit(reftest.run_tests(args[0], options))
sys.exit(reftest.run_tests(options.tests, options))

471
layout/tools/reftest/mach_commands.py
View File

@ -23,8 +23,7 @@ from mach.decorators import (
Command,
)
DEBUGGER_HELP = 'Debugger binary to run test in. Program name or path.'
import reftestcommandline
ADB_NOT_FOUND = '''
The %s command requires the adb binary to be on your path.
@ -81,43 +80,10 @@ class ReftestRunner(MozbuildObject):
self.tests_dir = os.path.join(self.topobjdir, '_tests')
self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
def _manifest_file(self, suite):
"""Returns the manifest file used for a given test suite."""
files = {
'reftest': 'reftest.list',
'reftest-ipc': 'reftest.list',
'crashtest': 'crashtests.list',
'crashtest-ipc': 'crashtests.list',
'jstestbrowser': 'jstests.list'
}
assert suite in files
return files[suite]
def _find_manifest(self, suite, test_file):
"""Return a tuple of (manifest-path, filter-string) for running test_file.
test_file can be a relative path to a single test file or manifest from
the top source directory, an absolute path to the same, or a directory
containing a manifest.
"""
assert test_file
path_arg = self._wrap_path_argument(test_file)
relpath = path_arg.relpath()
if os.path.isdir(path_arg.srcdir_path()):
return (mozpath.join(relpath, self._manifest_file(suite)), None)
if relpath.endswith('.list'):
return (relpath, None)
return (self._find_manifest(suite, mozpath.dirname(test_file))[0],
mozpath.basename(test_file))
def _make_shell_string(self, s):
return "'%s'" % re.sub("'", r"'\''", s)
def run_b2g_test(self, b2g_home=None, xre_path=None, test_file=None,
suite=None, filter=None, **kwargs):
def run_b2g_test(self, b2g_home=None, xre_path=None, **kwargs):
"""Runs a b2g reftest.
filter is a regular expression (in JS syntax, as could be passed to the
@ -130,324 +96,175 @@ class ReftestRunner(MozbuildObject):
suite is the type of reftest to run. It can be one of ('reftest',
'crashtest').
"""
if suite not in ('reftest', 'crashtest'):
if kwargs["suite"] not in ('reftest', 'crashtest'):
raise Exception('None or unrecognized reftest suite type.')
sys.path.insert(0, self.reftest_dir)
test_subdir = {"reftest": os.path.join('layout', 'reftests'),
"crashtest": os.path.join('layout', 'crashtest')}[kwargs["suite"]]
# Find the manifest file
if not test_file:
if suite == 'reftest':
test_file = mozpath.join('layout', 'reftests')
elif suite == 'crashtest':
test_file = mozpath.join('testing', 'crashtest')
if not kwargs["tests"]:
if not os.path.exists(os.path.join(self.topsrcdir, test_subdir)):
test_file = mozpath.relpath(os.path.abspath(test_subdir),
self.topsrcdir)
kwargs["tests"] = [test_subdir]
if not os.path.exists(os.path.join(self.topsrcdir, test_file)):
test_file = mozpath.relpath(os.path.abspath(test_file),
self.topsrcdir)
(manifest, single_file_filter) = self._find_manifest(suite, test_file)
if not os.path.exists(mozpath.join(self.topsrcdir, manifest)):
raise Exception('No manifest file was found at %s.' % manifest)
if single_file_filter:
if filter:
raise Exception('Cannot run single files in conjunction with --filter')
filter = single_file_filter
# Need to chdir to reftest_dir otherwise imports fail below.
os.chdir(self.reftest_dir)
# The imp module can spew warnings if the modules below have
# already been imported, ignore them.
with warnings.catch_warnings():
warnings.simplefilter('ignore')
import imp
path = os.path.join(self.reftest_dir, 'runreftestb2g.py')
with open(path, 'r') as fh:
imp.load_module('reftest', fh, path, ('.py', 'r', imp.PY_SOURCE))
import reftest
# Set up the reftest options.
parser = reftest.B2GOptions()
options, args = parser.parse_args([])
# Tests need to be served from a subdirectory of the server. Symlink
# topsrcdir here to get around this.
tests = os.path.join(self.reftest_dir, 'tests')
if not os.path.isdir(tests):
os.symlink(self.topsrcdir, tests)
args.insert(0, os.path.join('tests', manifest))
for k, v in kwargs.iteritems():
setattr(options, k, v)
for i, path in enumerate(kwargs["tests"]):
# Non-absolute paths are relative to the packaged directory, which
# has an extra tests/ at the start
if os.path.exists(os.path.abspath(path)):
path = os.path.relpath(path, os.path.join(self.topsrcdir))
kwargs["tests"][i] = os.path.join('tests', path)
if conditions.is_b2g_desktop(self):
if self.substs.get('ENABLE_MARIONETTE') != '1':
print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop',
self.mozconfig['path']))
return 1
return self.run_b2g_desktop(**kwargs)
options.profile = options.profile or os.environ.get('GAIA_PROFILE')
if not options.profile:
return self.run_b2g_remote(b2g_home, xre_path, **kwargs)
def run_b2g_desktop(self, **kwargs):
if self.substs.get('ENABLE_MARIONETTE') != '1':
print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop',
self.mozconfig['path']))
return 1
if not kwargs["profile"]:
gaia_profile = os.environ.get('GAIA_PROFILE')
if not gaia_profile:
print(GAIA_PROFILE_NOT_FOUND % 'reftest-b2g-desktop')
return 1
kwargs["profile"] = gaia_profile
if os.path.isfile(os.path.join(options.profile, 'extensions', \
'httpd@gaiamobile.org')):
print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop',
options.profile))
return 1
options.desktop = True
options.app = self.get_binary_path()
if options.oop:
options.browser_arg = '-oop'
if not options.app.endswith('-bin'):
options.app = '%s-bin' % options.app
if not os.path.isfile(options.app):
options.app = options.app[:-len('-bin')]
if os.path.isfile(os.path.join(kwargs["profile"], 'extensions',
'httpd@gaiamobile.org')):
print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop',
kwargs["profile"]))
return 1
return reftest.run_desktop_reftests(parser, options, args)
kwargs["desktop"] = True
kwargs["app"] = self.get_binary_path()
if kwargs["oop"]:
options.browser_arg = '-oop'
if not kwargs["app"].endswith('-bin'):
kwargs["app"] = '%s-bin' % options.app
if not os.path.isfile(kwargs["app"]):
options.app = kwargs["app"][:-len('-bin')]
return runreftestb2g.run(**kwargs)
def run_b2g_remote(self, b2g_home, xre_path, **kwargs):
import runreftestb2g
try:
which.which('adb')
except which.WhichError:
# TODO Find adb automatically if it isn't on the path
raise Exception(ADB_NOT_FOUND % ('%s-remote' % suite, b2g_home))
raise Exception(ADB_NOT_FOUND % ('%s-remote' % kwargs["suite"], b2g_home))
options.b2gPath = b2g_home
options.logdir = self.reftest_dir
options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
options.xrePath = xre_path
options.ignoreWindowSize = True
options.filter = filter
kwargs["b2gPath"] = b2g_home
kwargs["logdir"] = self.reftest_dir
kwargs["httpdPath"] = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
kwargs["xrePath"] = xre_path
kwargs["ignoreWindowSize"] = True
# Don't enable oop for crashtest until they run oop in automation
if suite == 'reftest':
options.oop = True
if kwargs["suite"] == 'reftest':
kwargs["oop"] = True
return reftest.run_remote_reftests(parser, options, args)
return runreftestb2g.run_remote(**kwargs)
def run_desktop_test(self, test_file=None, filter=None, suite=None,
debugger=None, debugger_args=None, parallel=False, shuffle=False,
e10s=False, extraPrefs=None, this_chunk=None, total_chunks=None):
"""Runs a reftest.
def run_desktop_test(self, **kwargs):
"""Runs a reftest."""
import runreftest
test_file is a path to a test file. It can be a relative path from the
top source directory, an absolute filename, or a directory containing
test files.
filter is a regular expression (in JS syntax, as could be passed to the
RegExp constructor) to select which reftests to run from the manifest.
suite is the type of reftest to run. It can be one of ('reftest',
'crashtest', 'jstestbrowser').
debugger is the program name (in $PATH) or the full path of the
debugger to run.
debugger_args are the arguments passed to the debugger.
parallel indicates whether tests should be run in parallel or not.
shuffle indicates whether to run tests in random order.
"""
if suite not in ('reftest', 'reftest-ipc', 'crashtest', 'crashtest-ipc', 'jstestbrowser'):
if kwargs["suite"] not in ('reftest', 'crashtest', 'jstestbrowser'):
raise Exception('None or unrecognized reftest suite type.')
env = {}
extra_args = []
default_manifest = {
"reftest": (self.topsrcdir, "layout", "reftests", "reftest.list"),
"crashtest": (self.topsrcdir, "testing", "crashtest", "crashtests.list"),
"jstestbrowser": (self.topobjdir, "dist", "test-stage", "jsreftest", "tests",
"jstests.list")
}
if test_file:
(path, single_file_filter) = self._find_manifest(suite, test_file)
if not os.path.exists(mozpath.join(self.topsrcdir, path)):
raise Exception('No manifest file was found at %s.' % path)
if single_file_filter:
if filter:
raise Exception('Cannot run single files in conjunction with --filter')
filter = single_file_filter
env[b'TEST_PATH'] = path
if filter:
extra_args.extend(['--filter', self._make_shell_string(filter)])
kwargs["extraProfileFiles"] = [os.path.join(self.topobjdir, "dist", "plugins")]
kwargs["symbolsPath"] = os.path.join(self.topobjdir, "crashreporter-symbols")
pass_thru = False
if not kwargs["tests"]:
kwargs["tests"] = [os.path.join(*default_manifest[kwargs["suite"]])]
if debugger:
extra_args.append('--debugger=\'%s\'' % debugger)
pass_thru = True
if debugger_args:
# Use _make_shell_string (which quotes) so that we
# handle multiple args being passed to the debugger.
extra_args.extend(['--debugger-args', self._make_shell_string(debugger_args)])
else:
if debugger_args:
print("--debugger-args passed, but no debugger specified.")
return 1
if kwargs["suite"] == "jstestbrowser":
kwargs["extraProfileFiles"].append(os.path.join(self.topobjdir, "dist",
"test-stage", "jsreftest",
"tests", "user.js"))
if parallel:
extra_args.append('--run-tests-in-parallel')
if not kwargs["runTestsInParallel"]:
kwargs["logFile"] = "%s.log" % kwargs["suite"]
if shuffle:
extra_args.append('--shuffle')
if e10s:
extra_args.append('--e10s')
if extraPrefs:
for pref in extraPrefs:
extra_args.extend(['--setpref', pref])
if this_chunk:
extra_args.append('--this-chunk=%s' % this_chunk)
if total_chunks:
extra_args.append('--total-chunks=%s' % total_chunks)
if extra_args:
args = [os.environ.get(b'EXTRA_TEST_ARGS', '')]
args.extend(extra_args)
env[b'EXTRA_TEST_ARGS'] = ' '.join(args)
# TODO hook up harness via native Python
return self._run_make(directory='.', target=suite, append_env=env,
pass_thru=pass_thru, ensure_exit_code=False)
def ReftestCommand(func):
"""Decorator that adds shared command arguments to reftest commands."""
debugger = CommandArgument('--debugger', metavar='DEBUGGER',
help=DEBUGGER_HELP)
func = debugger(func)
debugger_args = CommandArgument('--debugger-args', metavar='DEBUGGER_ARGS',
help='Arguments to pass to the debugger.')
func = debugger_args(func)
flter = CommandArgument('--filter', metavar='REGEX',
help='A JS regular expression to match test URLs against, to select '
'a subset of tests to run.')
func = flter(func)
path = CommandArgument('test_file', nargs='?', metavar='MANIFEST',
help='Reftest manifest file, or a directory in which to select '
'reftest.list. If omitted, the entire test suite is executed.')
func = path(func)
parallel = CommandArgument('--parallel', action='store_true',
help='Run tests in parallel.')
func = parallel(func)
shuffle = CommandArgument('--shuffle', action='store_true',
help='Run tests in random order.')
func = shuffle(func)
e10s = CommandArgument('--e10s', action='store_true',
help='Use content processes.')
func = e10s(func)
extraPrefs = CommandArgument('--setpref', action='append',
default=[], dest='extraPrefs', metavar='PREF=VALUE',
help='Set prefs in the reftest profile.')
func = extraPrefs(func)
totalChunks = CommandArgument('--total-chunks',
help = 'How many chunks to split the tests up into.')
func = totalChunks(func)
thisChunk = CommandArgument('--this-chunk',
help = 'Which chunk to run between 1 and --total-chunks.')
func = thisChunk(func)
return func
def B2GCommand(func):
"""Decorator that adds shared command arguments to b2g reftest commands."""
busybox = CommandArgument('--busybox', default=None,
help='Path to busybox binary to install on device')
func = busybox(func)
logdir = CommandArgument('--logdir', default=None,
help='directory to store log files')
func = logdir(func)
sdcard = CommandArgument('--sdcard', default="10MB",
help='Define size of sdcard: 1MB, 50MB...etc')
func = sdcard(func)
emulator_res = CommandArgument('--emulator-res', default='800x1000',
help='Emulator resolution of the format \'<width>x<height>\'')
func = emulator_res(func)
marionette = CommandArgument('--marionette', default=None,
help='host:port to use when connecting to Marionette')
func = marionette(func)
totalChunks = CommandArgument('--total-chunks', dest='totalChunks',
type = int,
help = 'How many chunks to split the tests up into.')
func = totalChunks(func)
thisChunk = CommandArgument('--this-chunk', dest='thisChunk',
type = int,
help = 'Which chunk to run between 1 and --total-chunks.')
func = thisChunk(func)
flter = CommandArgument('--filter', metavar='REGEX',
help='A JS regular expression to match test URLs against, to select '
'a subset of tests to run.')
func = flter(func)
oop = CommandArgument('--enable-oop', action='store_true', dest='oop',
help = 'Run tests in out-of-process mode.')
func = oop(func)
path = CommandArgument('test_file', default=None, nargs='?',
metavar='TEST',
help='Test to run. Can be specified as a single file, a ' \
'directory, or omitted. If omitted, the entire test suite is ' \
'executed.')
func = path(func)
return func
#Remove the stdout handler from the internal logger and let mach deal with it
runreftest.log.removeHandler(runreftest.log.handlers[0])
self.log_manager.enable_unstructured()
rv = runreftest.run(**kwargs)
self.log_manager.disable_unstructured()
return rv
@CommandProvider
class MachCommands(MachCommandBase):
@Command('reftest', category='testing', description='Run reftests (layout and graphics correctness).')
@ReftestCommand
def run_reftest(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='reftest', **kwargs)
@Command('reftest',
category='testing',
description='Run reftests (layout and graphics correctness).',
parser=reftestcommandline.DesktopArgumentsParser)
def run_reftest(self, **kwargs):
kwargs["suite"] = "reftest"
return self._run_reftest(**kwargs)
@Command('jstestbrowser', category='testing',
description='Run js/src/tests in the browser.')
@ReftestCommand
def run_jstestbrowser(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='jstestbrowser', **kwargs)
@Command('jstestbrowser',
category='testing',
description='Run js/src/tests in the browser.',
parser=reftestcommandline.DesktopArgumentsParser)
def run_jstestbrowser(self, **kwargs):
self._mach_context.commands.dispatch("build",
self._mach_context,
what=["stage-jstests"])
kwargs["suite"] = "jstestbrowser"
return self._run_reftest(**kwargs)
@Command('reftest-ipc', category='testing',
description='Run IPC reftests (layout and graphics correctness, separate process).')
@ReftestCommand
def run_ipc(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='reftest-ipc', **kwargs)
@Command('reftest-ipc',
category='testing',
description='Run IPC reftests (layout and graphics correctness, separate process).',
parser=reftestcommandline.DesktopArgumentsParser)
def run_ipc(self, **kwargs):
kwargs["ipc"] = True
kwargs["suite"] = "reftest"
return self._run_reftest(**kwargs)
@Command('crashtest', category='testing',
description='Run crashtests (Check if crashes on a page).')
@ReftestCommand
def run_crashtest(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='crashtest', **kwargs)
@Command('crashtest',
category='testing',
description='Run crashtests (Check if crashes on a page).',
parser=reftestcommandline.DesktopArgumentsParser)
def run_crashtest(self, **kwargs):
kwargs["suite"] = "crashtest"
return self._run_reftest(**kwargs)
@Command('crashtest-ipc', category='testing',
description='Run IPC crashtests (Check if crashes on a page, separate process).')
@ReftestCommand
def run_crashtest_ipc(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='crashtest-ipc', **kwargs)
@Command('crashtest-ipc',
category='testing',
description='Run IPC crashtests (Check if crashes on a page, separate process).',
parser=reftestcommandline.DesktopArgumentsParser)
def run_crashtest_ipc(self, **kwargs):
kwargs["ipc"] = True
kwargs["suite"] = "crashtest"
return self._run_reftest(**kwargs)
def _run_reftest(self, test_file=None, suite=None, **kwargs):
def _run_reftest(self, **kwargs):
reftest = self._spawn(ReftestRunner)
return reftest.run_desktop_test(test_file, suite=suite, **kwargs)
return reftest.run_desktop_test(**kwargs)
# TODO For now b2g commands will only work with the emulator,
@ -466,27 +283,30 @@ class B2GCommands(MachCommandBase):
setattr(self, attr, getattr(context, attr, None))
@Command('reftest-remote', category='testing',
description='Run a remote reftest (b2g layout and graphics correctness, remote device).',
conditions=[conditions.is_b2g, is_emulator])
@B2GCommand
def run_reftest_remote(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='reftest', **kwargs)
description='Run a remote reftest (b2g layout and graphics correctness, remote device).',
conditions=[conditions.is_b2g, is_emulator],
parser=reftestcommandline.B2GArgumentParser)
def run_reftest_remote(self, **kwargs):
kwargs["suite"] = "reftest"
return self._run_reftest(**kwargs)
@Command('reftest-b2g-desktop', category='testing',
description='Run a b2g desktop reftest (b2g desktop layout and graphics correctness).',
conditions=[conditions.is_b2g_desktop])
@B2GCommand
def run_reftest_b2g_desktop(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='reftest', **kwargs)
description='Run a b2g desktop reftest (b2g desktop layout and graphics correctness).',
conditions=[conditions.is_b2g_desktop],
parser=reftestcommandline.B2GArgumentParser)
def run_reftest_b2g_desktop(self, **kwargs):
kwargs["suite"] = "reftest"
return self._run_reftest(**kwargs)
@Command('crashtest-remote', category='testing',
description='Run a remote crashtest (Check if b2g crashes on a page, remote device).',
conditions=[conditions.is_b2g, is_emulator])
@B2GCommand
description='Run a remote crashtest (Check if b2g crashes on a page, remote device).',
conditions=[conditions.is_b2g, is_emulator],
parser=reftestcommandline.B2GArgumentParser)
def run_crashtest_remote(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='crashtest', **kwargs)
kwargs["suite"] = "crashtest"
return self._run_reftest(**kwargs)
def _run_reftest(self, test_file=None, suite=None, **kwargs):
def _run_reftest(self, **kwargs):
if self.device_name:
if self.device_name.startswith('emulator'):
emulator = 'arm'
@ -495,5 +315,4 @@ class B2GCommands(MachCommandBase):
kwargs['emulator'] = emulator
reftest = self._spawn(ReftestRunner)
return reftest.run_b2g_test(self.b2g_home, self.xre_path,
test_file, suite=suite, **kwargs)
return reftest.run_b2g_test(self.b2g_home, self.xre_path, **kwargs)

62
layout/tools/reftest/reftest.js
View File

@ -48,7 +48,7 @@ var gShuffle = false;
var gTotalChunks = 0;
var gThisChunk = 0;
var gContainingWindow = null;
var gURLFilterRegex = null;
var gURLFilterRegex = {};
const FOCUS_FILTER_ALL_TESTS = "all";
const FOCUS_FILTER_NEEDS_FOCUS_TESTS = "needs-focus";
const FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS = "non-needs-focus";
@ -389,10 +389,6 @@ function InitAndStartRefTests()
gThisChunk = 0;
}
try {
gURLFilterRegex = new RegExp(prefs.getCharPref("reftest.filter"));
} catch(e) {}
try {
gFocusFilterMode = prefs.getCharPref("reftest.focusFilterMode");
} catch(e) {}
@ -449,7 +445,7 @@ function Shuffle(array)
function StartTests()
{
var uri;
var manifests;
/* These prefs are optional, so we don't need to spit an error to the log */
try {
var prefs = Components.classes["@mozilla.org/preferences-service;1"].
@ -476,23 +472,29 @@ function StartTests()
gRunSlowTests = false;
}
try {
uri = prefs.getCharPref("reftest.uri");
} catch(e) {
uri = "";
}
if (uri == "") {
gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | Unable to find reftest.uri pref. Please ensure your profile is setup properly\n");
DoneTests();
}
if (gShuffle) {
gNoCanvasCache = true;
}
gURLs = [];
try {
ReadTopManifest(uri);
var manifests = JSON.parse(prefs.getCharPref("reftest.manifests"));
gURLFilterRegex = manifests[null];
} catch(e) {
gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | Unable to find reftest.manifests pref. Please ensure your profile is setup properly\n");
DoneTests();
}
try {
var globalFilter = manifests.hasOwnProperty("") ? new RegExp(manifests[""]) : null;
var manifestURLs = Object.keys(manifests);
manifestURLs.sort();
manifestURLs.forEach(function(manifestURL) {
gDumpLog("Readings manifest" + manifestURL + "\n");
var pathFilters = manifests[manifestURL].map(function(x) {return new RegExp(x)});
ReadTopManifest(manifestURL, [globalFilter, pathFilters]);
});
BuildUseCounts();
// Filter tests which will be skipped to get a more even distribution when chunking
@ -745,18 +747,22 @@ function AddPrefSettings(aWhere, aPrefName, aPrefValExpression, aSandbox, aTestP
return true;
}
function ReadTopManifest(aFileURL)
function ReadTopManifest(aFileURL, aFilters)
{
gURLs = new Array();
var url = gIOService.newURI(aFileURL, null, null);
if (!url)
throw "Expected a file or http URL for the manifest.";
ReadManifest(url, EXPECTED_PASS);
ReadManifest(url, EXPECTED_PASS, aFilters);
}
function AddTestItem(aTest)
function AddTestItem(aTest, aFilters)
{
if (gURLFilterRegex && !gURLFilterRegex.test(aTest.url1.spec))
if (!aFilters)
aFilters = [null, []];
if ((aFilters[0] && !aFilters[0].test(aTest.url1.spec)) ||
(aFilters[1].length > 0 &&
!aFilters[1].some(function(filter) {return filter.test(aTest.url1.spec)})))
return;
if (gFocusFilterMode == FOCUS_FILTER_NEEDS_FOCUS_TESTS &&
!aTest.needsFocus)
@ -769,7 +775,7 @@ function AddTestItem(aTest)
// Note: If you materially change the reftest manifest parsing,
// please keep the parser in print-manifest-dirs.py in sync.
function ReadManifest(aURL, inherited_status)
function ReadManifest(aURL, inherited_status, aFilters)
{
var secMan = CC[NS_SCRIPTSECURITYMANAGER_CONTRACTID]
.getService(CI.nsIScriptSecurityManager);
@ -986,7 +992,7 @@ function ReadManifest(aURL, inherited_status)
var incURI = gIOService.newURI(items[1], null, listURL);
secMan.checkLoadURIWithPrincipal(principal, incURI,
CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
ReadManifest(incURI, expected_status);
ReadManifest(incURI, expected_status, aFilters);
} else if (items[0] == TYPE_LOAD) {
if (items.length != 2)
throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to load";
@ -1016,7 +1022,7 @@ function ReadManifest(aURL, inherited_status)
fuzzyMaxPixels: fuzzy_max_pixels,
url1: testURI,
url2: null,
chaosMode: chaosMode });
chaosMode: chaosMode }, aFilters);
} else if (items[0] == TYPE_SCRIPT) {
if (items.length != 2)
throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to script";
@ -1043,7 +1049,7 @@ function ReadManifest(aURL, inherited_status)
fuzzyMaxPixels: fuzzy_max_pixels,
url1: testURI,
url2: null,
chaosMode: chaosMode });
chaosMode: chaosMode }, aFilters);
} else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL) {
if (items.length != 3)
throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to " + items[0];
@ -1073,7 +1079,7 @@ function ReadManifest(aURL, inherited_status)
fuzzyMaxPixels: fuzzy_max_pixels,
url1: testURI,
url2: refURI,
chaosMode: chaosMode });
chaosMode: chaosMode }, aFilters);
} else {
throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": unknown test type " + items[0];
}

740
layout/tools/reftest/reftestcommandline.py Normal file
View File

@ -0,0 +1,740 @@
import argparse
import os
from collections import OrderedDict
from urlparse import urlparse
here = os.path.abspath(os.path.dirname(__file__))
class ReftestArgumentsParser(argparse.ArgumentParser):
def __init__(self, **kwargs):
super(ReftestArgumentsParser, self).__init__(**kwargs)
# Try to import a MozbuildObject. Success indicates that we are
# running from a source tree. This allows some defaults to be set
# from the source tree.
try:
from mozbuild.base import MozbuildObject
self.build_obj = MozbuildObject.from_environment(cwd=here)
except ImportError:
self.build_obj = None
self.add_argument("--xre-path",
action="store",
type=str,
dest="xrePath",
# individual scripts will set a sane default
default=None,
help="absolute path to directory containing XRE (probably xulrunner)")
self.add_argument("--symbols-path",
action="store",
type=str,
dest="symbolsPath",
default=None,
help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
self.add_argument("--debugger",
action="store",
dest="debugger",
help="use the given debugger to launch the application")
self.add_argument("--debugger-args",
action="store",
dest="debuggerArgs",
help="pass the given args to the debugger _before_ "
"the application on the command line")
self.add_argument("--debugger-interactive",
action="store_true",
dest="debuggerInteractive",
help="prevents the test harness from redirecting "
"stdout and stderr for interactive debuggers")
self.add_argument("--appname",
action="store",
type=str,
dest="app",
default=None,
help="absolute path to application, overriding default")
self.add_argument("--extra-profile-file",
action="append",
dest="extraProfileFiles",
default=[],
help="copy specified files/dirs to testing profile")
self.add_argument("--timeout",
action="store",
dest="timeout",
type=int,
default=5 * 60, # 5 minutes per bug 479518
help="reftest will timeout in specified number of seconds. [default %(default)s].")
self.add_argument("--leak-threshold",
action="store",
type=int,
dest="defaultLeakThreshold",
default=0,
help="fail if the number of bytes leaked in default "
"processes through refcounted objects (or bytes "
"in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) "
"is greater than the given number")
self.add_argument("--utility-path",
action="store",
type=str,
dest="utilityPath",
default="bindir",
help="absolute path to directory containing utility "
"programs (xpcshell, ssltunnel, certutil)")
self.add_argument("--total-chunks",
type=int,
dest="totalChunks",
help="how many chunks to split the tests up into")
self.add_argument("--this-chunk",
type=int,
dest="thisChunk",
help="which chunk to run between 1 and --total-chunks")
self.add_argument("--log-file",
action="store",
type=str,
dest="logFile",
default=None,
help="file to log output to in addition to stdout")
self.add_argument("--skip-slow-tests",
dest="skipSlowTests",
action="store_true",
default=False,
help="skip tests marked as slow when running")
self.add_argument("--ignore-window-size",
dest="ignoreWindowSize",
action="store_true",
default=False,
help="ignore the window size, which may cause spurious failures and passes")
self.add_argument("--install-extension",
action="append",
dest="extensionsToInstall",
default=[],
help="install the specified extension in the testing profile. "
"The extension file's name should be <id>.xpi where <id> is "
"the extension's id as indicated in its install.rdf. "
"An optional path can be specified too.")
self.add_argument("--setenv",
action="append",
type=str,
default=[],
dest="environment",
metavar="NAME=VALUE",
help="sets the given variable in the application's "
"environment")
self.add_argument("--filter",
action="store",
type=str,
dest="filter",
help="specifies a regular expression (as could be passed to the JS "
"RegExp constructor) to test against URLs in the reftest manifest; "
"only test items that have a matching test URL will be run.")
self.add_argument("--shuffle",
action="store_true",
default=False,
dest="shuffle",
help="run reftests in random order")
self.add_argument("--focus-filter-mode",
action="store",
type=str,
dest="focusFilterMode",
default="all",
help="filters tests to run by whether they require focus. "
"Valid values are `all', `needs-focus', or `non-needs-focus'. "
"Defaults to `all'.")
self.add_argument("--e10s",
action="store_true",
default=False,
dest="e10s",
help="enables content processes")
self.add_argument("--setpref",
action="append",
type=str,
default=[],
dest="extraPrefs",
metavar="PREF=VALUE",
help="defines an extra user preference")
self.add_argument("--reftest-extension-path",
action="store",
dest="reftestExtensionPath",
help="Path to the reftest extension")
self.add_argument("--special-powers-extension-path",
action="store",
dest="specialPowersExtensionPath",
help="Path to the special powers extension")
self.add_argument("--suite",
choices=["reftest", "crashtest", "jstestbrowser"],
default=None,
help=argparse.SUPPRESS)
self.add_argument("tests",
metavar="TEST_PATH",
nargs="*",
help="Path to test file, manifest file, or directory containing tests")
def get_ip(self):
import moznetwork
if os.name != "nt":
return moznetwork.get_ip()
else:
self.error(
"ERROR: you must specify a --remote-webserver=<ip address>\n")
def set_default_suite(self, options):
manifests = OrderedDict([("reftest.list", "reftest"),
("crashtests.list", "crashtest"),
("jstests.list", "jstestbrowser")])
for test_path in options.tests:
file_name = os.path.basename(test_path)
if file_name in manifests:
options.suite = manifests[file_name]
return
for test_path in options.tests:
for manifest_file, suite in manifests.iteritems():
if os.path.exists(os.path.join(test_path, manifest_file)):
options.suite = suite
return
self.error("Failed to determine test suite; supply --suite to set this explicitly")
def validate(self, options, reftest):
import sys
if not options.tests:
# Can't just set this in the argument parser because mach will set a default
self.error("Must supply at least one path to a manifest file, test directory, or test file to run.")
if options.suite is None:
self.set_default_suite(options)
if options.totalChunks is not None and options.thisChunk is None:
self.error(
"thisChunk must be specified when totalChunks is specified")
if options.totalChunks:
if not 1 <= options.thisChunk <= options.totalChunks:
self.error("thisChunk must be between 1 and totalChunks")
if options.logFile:
options.logFile = reftest.getFullPath(options.logFile)
if options.xrePath is not None:
if not os.access(options.xrePath, os.F_OK):
self.error("--xre-path '%s' not found" % options.xrePath)
if not os.path.isdir(options.xrePath):
self.error("--xre-path '%s' is not a directory" %
options.xrePath)
options.xrePath = reftest.getFullPath(options.xrePath)
if options.reftestExtensionPath is None:
if self.build_obj is not None:
options.reftestExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests",
"reftest", "reftest")
else:
options.reftestExtensionPath = os.path.join(here, "reftest")
if (options.specialPowersExtensionPath is None and
options.suite in ["crashtest", "jstestbrowser"]):
if self.build_obj is not None:
options.specialPowersExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests",
"reftest", "specialpowers")
else:
options.specialPowersExtensionPath = os.path.join(
here, "specialpowers")
options.leakThresholds = {
"default": options.defaultLeakThreshold,
"tab": 5000, # See dependencies of bug 1051230.
}
class DesktopArgumentsParser(ReftestArgumentsParser):
def __init__(self, **kwargs):
super(DesktopArgumentsParser, self).__init__(**kwargs)
self.add_argument("--run-tests-in-parallel",
action="store_true",
default=False,
dest="runTestsInParallel",
help="run tests in parallel if possible")
self.add_argument("--ipc",
action="store_true",
default=False,
help="Run in out-of-processes mode")
def _prefs_oop(self):
import mozinfo
prefs = ["layers.async-pan-zoom.enabled=true",
"browser.tabs.remote.autostart=true"]
if mozinfo.os == "win":
prefs.append("layers.acceleration.disabled=true")
return prefs
def _prefs_gpu(self):
if mozinfo.os != "win":
return ["layers.acceleration.force-enabled=true"]
return []
def validate(self, options, reftest):
super(DesktopArgumentsParser, self).validate(options, reftest)
if options.ipc:
for item in self._prefs_oop():
if item not in options.extraPrefs:
options.extraPrefs.append(item)
if options.runTestsInParallel:
if options.logFile is not None:
self.error("cannot specify logfile with parallel tests")
if options.totalChunks is not None or options.thisChunk is not None:
self.error(
"cannot specify thisChunk or totalChunks with parallel tests")
if options.focusFilterMode != "all":
self.error("cannot specify focusFilterMode with parallel tests")
if options.debugger is not None:
self.error("cannot specify a debugger with parallel tests")
if not options.tests:
self.error("No test files specified.")
if options.app is None:
bin_dir = (self.build_obj.get_binary_path() if
self.build_obj and self.build_obj.substs[
'MOZ_BUILD_APP'] != 'mobile/android'
else None)
if bin_dir:
options.app = bin_dir
else:
self.error(
"could not find the application path, --appname must be specified")
options.app = reftest.getFullPath(options.app)
if not os.path.exists(options.app):
self.error("""Error: Path %(app)s doesn't exist.
Are you executing $objdir/_tests/reftest/runreftest.py?"""
% {"app": options.app})
if options.xrePath is None:
options.xrePath = os.path.dirname(options.app)
if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2:
options.symbolsPath = reftest.getFullPath(options.symbolsPath)
options.utilityPath = reftest.getFullPath(options.utilityPath)
class B2GArgumentParser(ReftestArgumentsParser):
def __init__(self, **kwargs):
super(B2GArgumentParser, self).__init__(**kwargs)
self.add_argument("--browser-arg",
action="store",
type=str,
dest="browser_arg",
help="Optional command-line arg to pass to the browser")
self.add_argument("--b2gpath",
action="store",
type=str,
dest="b2gPath",
help="path to B2G repo or qemu dir")
self.add_argument("--marionette",
action="store",
type=str,
dest="marionette",
help="host:port to use when connecting to Marionette")
self.add_argument("--emulator",
action="store",
type=str,
dest="emulator",
help="Architecture of emulator to use: x86 or arm")
self.add_argument("--emulator-res",
action="store",
type=str,
dest="emulator_res",
help="Emulator resolution of the format '<width>x<height>'")
self.add_argument("--no-window",
action="store_true",
dest="noWindow",
default=False,
help="Pass --no-window to the emulator")
self.add_argument("--adbpath",
action="store",
type=str,
dest="adb_path",
default="adb",
help="path to adb")
self.add_argument("--deviceIP",
action="store",
type=str,
dest="deviceIP",
help="ip address of remote device to test")
self.add_argument("--devicePort",
action="store",
type=str,
dest="devicePort",
default="20701",
help="port of remote device to test")
self.add_argument("--remote-logfile",
action="store",
type=str,
dest="remoteLogFile",
help="Name of log file on the device relative to the device root. PLEASE ONLY USE A FILENAME.")
self.add_argument("--remote-webserver",
action="store",
type=str,
dest="remoteWebServer",
help="ip address where the remote web server is hosted at")
self.add_argument("--http-port",
action="store",
type=str,
dest="httpPort",
help="ip address where the remote web server is hosted at")
self.add_argument("--ssl-port",
action="store",
type=str,
dest="sslPort",
help="ip address where the remote web server is hosted at")
self.add_argument("--pidfile",
action="store",
type=str,
dest="pidFile",
default="",
help="name of the pidfile to generate")
self.add_argument("--gecko-path",
action="store",
type=str,
dest="geckoPath",
help="the path to a gecko distribution that should "
"be installed on the emulator prior to test")
self.add_argument("--logdir",
action="store",
type=str,
dest="logdir",
help="directory to store log files")
self.add_argument('--busybox',
action='store',
type=str,
dest='busybox',
help="Path to busybox binary to install on device")
self.add_argument("--httpd-path",
action="store",
type=str,
dest="httpdPath",
help="path to the httpd.js file")
self.add_argument("--profile",
action="store",
type=str,
dest="profile",
help="for desktop testing, the path to the "
"gaia profile to use")
self.add_argument("--desktop",
action="store_true",
dest="desktop",
default=False,
help="Run the tests on a B2G desktop build")
self.add_argument("--mulet",
action="store_true",
dest="mulet",
default=False,
help="Run the tests on a B2G desktop build")
self.add_argument("--enable-oop",
action="store_true",
dest="oop",
default=False,
help="Run the tests out of process")
self.set_defaults(remoteTestRoot=None,
logFile="reftest.log",
autorun=True,
closeWhenDone=True,
testPath="")
def validate_remote(self, options, automation):
if not options.app:
options.app = automation.DEFAULT_APP
if not options.remoteTestRoot:
options.remoteTestRoot = automation._devicemanager.deviceRoot + \
"/reftest"
options.remoteProfile = options.remoteTestRoot + "/profile"
productRoot = options.remoteTestRoot + "/" + automation._product
if options.utilityPath is None:
options.utilityPath = productRoot + "/bin"
if not options.httpPort:
options.httpPort = automation.DEFAULT_HTTP_PORT
if not options.sslPort:
options.sslPort = automation.DEFAULT_SSL_PORT
if options.remoteWebServer is None:
options.remoteWebServer = self.get_ip()
options.webServer = options.remoteWebServer
if options.geckoPath and not options.emulator:
self.error(
"You must specify --emulator if you specify --gecko-path")
if options.logdir and not options.emulator:
self.error("You must specify --emulator if you specify --logdir")
if options.remoteLogFile is None:
options.remoteLogFile = "reftest.log"
options.localLogName = options.remoteLogFile
options.remoteLogFile = options.remoteTestRoot + \
'/' + options.remoteLogFile
# Ensure that the options.logfile (which the base class uses) is set to
# the remote setting when running remote. Also, if the user set the
# log file name there, use that instead of reusing the remotelogfile as
# above.
if (options.logFile):
# If the user specified a local logfile name use that
options.localLogName = options.logFile
options.logFile = options.remoteLogFile
# Only reset the xrePath if it wasn't provided
if options.xrePath is None:
options.xrePath = options.utilityPath
options.xrePath = os.path.abspath(options.xrePath)
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# httpd-path is specified by standard makefile targets and may be specified
# on the command line to select a particular version of httpd.js. If not
# specified, try to select the one from from the xre bundle, as
# required in bug 882932.
if not options.httpdPath:
options.httpdPath = os.path.join(options.xrePath, "components")
return options
class RemoteArgumentsParser(ReftestArgumentsParser):
def __init__(self, **kwargs):
super(RemoteArgumentsParser, self).__init__()
# app, xrePath and utilityPath variables are set in main function
self.set_defaults(logFile="reftest.log",
app="",
xrePath="",
utilityPath="",
localLogName=None)
self.add_argument("--remote-app-path",
action="store",
type=str,
dest="remoteAppPath",
help="Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.")
self.add_argument("--deviceIP",
action="store",
type=str,
dest="deviceIP",
help="ip address of remote device to test")
self.add_argument("--deviceSerial",
action="store",
type=str,
dest="deviceSerial",
help="adb serial number of remote device to test")
self.add_argument("--devicePort",
action="store",
type=str,
default="20701",
dest="devicePort",
help="port of remote device to test")
self.add_argument("--remote-product-name",
action="store",
type=str,
dest="remoteProductName",
default="fennec",
help="Name of product to test - either fennec or firefox, defaults to fennec")
self.add_argument("--remote-webserver",
action="store",
type=str,
dest="remoteWebServer",
help="IP Address of the webserver hosting the reftest content")
self.add_argument("--http-port",
action="store",
type=str,
dest="httpPort",
help="port of the web server for http traffic")
self.add_argument("--ssl-port",
action="store",
type=str,
dest="sslPort",
help="Port for https traffic to the web server")
self.add_argument("--remote-logfile",
action="store",
type=str,
dest="remoteLogFile",
default="reftest.log",
help="Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.")
self.add_argument("--pidfile",
action="store",
type=str,
dest="pidFile",
default="",
help="name of the pidfile to generate")
self.add_argument("--bootstrap",
action="store_true",
dest="bootstrap",
default=False,
help="test with a bootstrap addon required for native Fennec")
self.add_argument("--dm_trans",
action="store",
type=str,
dest="dm_trans",
default="sut",
help="the transport to use to communicate with device: [adb|sut]; default=sut")
self.add_argument("--remoteTestRoot",
action="store",
type=str,
dest="remoteTestRoot",
help="remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
self.add_argument("--httpd-path",
action="store",
type=str,
dest="httpdPath",
help="path to the httpd.js file")
def validate_remote(self, options, automation):
# Ensure our defaults are set properly for everything we can infer
if not options.remoteTestRoot:
options.remoteTestRoot = automation._devicemanager.deviceRoot + \
'/reftest'
options.remoteProfile = options.remoteTestRoot + "/profile"
if options.remoteWebServer is None:
options.remoteWebServer = self.get_ip()
# Verify that our remotewebserver is set properly
if options.remoteWebServer == '127.0.0.1':
self.error("ERROR: Either you specified the loopback for the remote webserver or ",
"your local IP cannot be detected. Please provide the local ip in --remote-webserver")
if not options.httpPort:
options.httpPort = automation.DEFAULT_HTTP_PORT
if not options.sslPort:
options.sslPort = automation.DEFAULT_SSL_PORT
# One of remoteAppPath (relative path to application) or the app (executable) must be
# set, but not both. If both are set, we destroy the user's selection for app
# so instead of silently destroying a user specificied setting, we
# error.
if options.remoteAppPath and options.app:
self.error(
"ERROR: You cannot specify both the remoteAppPath and the app")
elif options.remoteAppPath:
options.app = options.remoteTestRoot + "/" + options.remoteAppPath
elif options.app is None:
# Neither remoteAppPath nor app are set -- error
self.error("ERROR: You must specify either appPath or app")
if options.xrePath is None:
self.error(
"ERROR: You must specify the path to the controller xre directory")
else:
# Ensure xrepath is a full path
options.xrePath = os.path.abspath(options.xrePath)
options.localLogName = options.remoteLogFile
options.remoteLogFile = options.remoteTestRoot + \
'/' + options.remoteLogFile
# Ensure that the options.logfile (which the base class uses) is set to
# the remote setting when running remote. Also, if the user set the
# log file name there, use that instead of reusing the remotelogfile as
# above.
if options.logFile:
# If the user specified a local logfile name use that
options.localLogName = options.logFile
options.logFile = options.remoteLogFile
if options.pidFile != "":
with open(options.pidFile, 'w') as f:
f.write(str(os.getpid()))
# httpd-path is specified by standard makefile targets and may be specified
# on the command line to select a particular version of httpd.js. If not
# specified, try to select the one from hostutils.zip, as required in
# bug 882932.
if not options.httpdPath:
options.httpdPath = os.path.join(options.utilityPath, "components")
if not options.ignoreWindowSize:
parts = automation._devicemanager.getInfo(
'screen')['screen'][0].split()
width = int(parts[0].split(':')[1])
height = int(parts[1].split(':')[1])
if (width < 1050 or height < 1050):
self.error("ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (
width, height))

222
layout/tools/reftest/remotereftest.py
View File

@ -9,10 +9,9 @@ import tempfile
import traceback
# We need to know our current directory so that we can serve our test files from it.
SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
from runreftest import RefTest
from runreftest import ReftestOptions
from runreftest import RefTest, ReftestResolver
from automation import Automation
import devicemanager
import droid
@ -20,159 +19,26 @@ import mozinfo
import moznetwork
from remoteautomation import RemoteAutomation, fennecLogcatFilters
class RemoteOptions(ReftestOptions):
def __init__(self, automation):
ReftestOptions.__init__(self)
self.automation = automation
import reftestcommandline
defaults = {}
defaults["logFile"] = "reftest.log"
# app, xrePath and utilityPath variables are set in main function
defaults["app"] = ""
defaults["xrePath"] = ""
defaults["utilityPath"] = ""
defaults["runTestsInParallel"] = False
self.add_option("--remote-app-path", action="store",
type = "string", dest = "remoteAppPath",
help = "Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.")
defaults["remoteAppPath"] = None
self.add_option("--deviceIP", action="store",
type = "string", dest = "deviceIP",
help = "ip address of remote device to test")
defaults["deviceIP"] = None
self.add_option("--deviceSerial", action="store",
type = "string", dest = "deviceSerial",
help = "adb serial number of remote device to test")
defaults["deviceSerial"] = None
self.add_option("--devicePort", action="store",
type = "string", dest = "devicePort",
help = "port of remote device to test")
defaults["devicePort"] = 20701
self.add_option("--remote-product-name", action="store",
type = "string", dest = "remoteProductName",
help = "Name of product to test - either fennec or firefox, defaults to fennec")
defaults["remoteProductName"] = "fennec"
self.add_option("--remote-webserver", action="store",
type = "string", dest = "remoteWebServer",
help = "IP Address of the webserver hosting the reftest content")
defaults["remoteWebServer"] = moznetwork.get_ip()
self.add_option("--http-port", action = "store",
type = "string", dest = "httpPort",
help = "port of the web server for http traffic")
defaults["httpPort"] = automation.DEFAULT_HTTP_PORT
self.add_option("--ssl-port", action = "store",
type = "string", dest = "sslPort",
help = "Port for https traffic to the web server")
defaults["sslPort"] = automation.DEFAULT_SSL_PORT
self.add_option("--remote-logfile", action="store",
type = "string", dest = "remoteLogFile",
help = "Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.")
defaults["remoteLogFile"] = None
self.add_option("--pidfile", action = "store",
type = "string", dest = "pidFile",
help = "name of the pidfile to generate")
defaults["pidFile"] = ""
self.add_option("--bootstrap", action="store_true", dest = "bootstrap",
help = "test with a bootstrap addon required for native Fennec")
defaults["bootstrap"] = False
self.add_option("--dm_trans", action="store",
type = "string", dest = "dm_trans",
help = "the transport to use to communicate with device: [adb|sut]; default=sut")
defaults["dm_trans"] = "sut"
self.add_option("--remoteTestRoot", action = "store",
type = "string", dest = "remoteTestRoot",
help = "remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
defaults["remoteTestRoot"] = None
self.add_option("--httpd-path", action = "store",
type = "string", dest = "httpdPath",
help = "path to the httpd.js file")
defaults["httpdPath"] = None
defaults["localLogName"] = None
self.set_defaults(**defaults)
def verifyRemoteOptions(self, options):
if options.runTestsInParallel:
self.error("Cannot run parallel tests here")
# Ensure our defaults are set properly for everything we can infer
if not options.remoteTestRoot:
options.remoteTestRoot = self.automation._devicemanager.deviceRoot + '/reftest'
options.remoteProfile = options.remoteTestRoot + "/profile"
# Verify that our remotewebserver is set properly
if (options.remoteWebServer == None or
options.remoteWebServer == '127.0.0.1'):
print "ERROR: Either you specified the loopback for the remote webserver or ",
print "your local IP cannot be detected. Please provide the local ip in --remote-webserver"
return None
# One of remoteAppPath (relative path to application) or the app (executable) must be
# set, but not both. If both are set, we destroy the user's selection for app
# so instead of silently destroying a user specificied setting, we error.
if (options.remoteAppPath and options.app):
print "ERROR: You cannot specify both the remoteAppPath and the app"
return None
elif (options.remoteAppPath):
options.app = options.remoteTestRoot + "/" + options.remoteAppPath
elif (options.app == None):
# Neither remoteAppPath nor app are set -- error
print "ERROR: You must specify either appPath or app"
return None
if (options.xrePath == None):
print "ERROR: You must specify the path to the controller xre directory"
return None
class RemoteReftestResolver(ReftestResolver):
def absManifestPath(self, path):
script_abs_path = os.path.join(SCRIPT_DIRECTORY, path)
if os.path.exists(script_abs_path):
rv = script_abs_path
elif os.path.exists(os.path.abspath(path)):
rv = os.path.abspath(path)
else:
# Ensure xrepath is a full path
options.xrePath = os.path.abspath(options.xrePath)
print >> sys.stderr, "Could not find manifest %s" % script_abs_path
sys.exit(1)
return os.path.normpath(rv)
# Default to <deviceroot>/reftest/reftest.log
if (options.remoteLogFile == None):
options.remoteLogFile = 'reftest.log'
def manifestURL(self, options, path):
# Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
# It's possible for this url to have a leading "..", but reftest.js will fix that up
relPath = os.path.relpath(path, SCRIPT_DIRECTORY)
return "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, relPath)
options.localLogName = options.remoteLogFile
options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
# Ensure that the options.logfile (which the base class uses) is set to
# the remote setting when running remote. Also, if the user set the
# log file name there, use that instead of reusing the remotelogfile as above.
if (options.logFile):
# If the user specified a local logfile name use that
options.localLogName = options.logFile
options.logFile = options.remoteLogFile
if (options.pidFile != ""):
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# httpd-path is specified by standard makefile targets and may be specified
# on the command line to select a particular version of httpd.js. If not
# specified, try to select the one from hostutils.zip, as required in bug 882932.
if not options.httpdPath:
options.httpdPath = os.path.join(options.utilityPath, "components")
# TODO: Copied from main, but I think these are no longer used in a post xulrunner world
#options.xrePath = options.remoteTestRoot + self.automation._product + '/xulrunner'
#options.utilityPath = options.testRoot + self.automation._product + '/bin'
return options
class ReftestServer:
""" Web server used to serve Reftests, for closer fidelity to the real web.
@ -260,6 +126,7 @@ class ReftestServer:
class RemoteReftest(RefTest):
remoteApp = ''
resolver_cls = RemoteReftestResolver
def __init__(self, automation, devicemanager, options, scriptDir):
RefTest.__init__(self)
@ -342,8 +209,12 @@ class RemoteReftest(RefTest):
def stopWebServer(self, options):
self.server.stop()
def createReftestProfile(self, options, reftestlist):
profile = RefTest.createReftestProfile(self, options, reftestlist, server=options.remoteWebServer, port=options.httpPort)
def createReftestProfile(self, options, manifest):
profile = RefTest.createReftestProfile(self,
options,
manifest,
server=options.remoteWebServer,
port=options.httpPort)
profileDir = profile.profile
prefs = {}
@ -355,7 +226,6 @@ class RemoteReftest(RefTest):
# Set a future policy version to avoid the telemetry prompt.
prefs["toolkit.telemetry.prompted"] = 999
prefs["toolkit.telemetry.notifiedOptOut"] = 999
prefs["reftest.uri"] = "%s" % reftestlist
prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
# Point the url-classifier to the local testing server for fast failures
@ -410,9 +280,6 @@ class RemoteReftest(RefTest):
print "Automation Error: Failed to copy extra files to device"
raise
def getManifestPath(self, path):
return path
def printDeviceInfo(self, printLogcat=False):
try:
if printLogcat:
@ -473,10 +340,10 @@ class RemoteReftest(RefTest):
except:
print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % self.pidFile
def main(args):
def main():
automation = RemoteAutomation(None)
parser = RemoteOptions(automation)
options, args = parser.parse_args()
parser = reftestcommandline.RemoteArgumentsParser()
options = parser.parse_args()
if (options.dm_trans == 'sut' and options.deviceIP == None):
print "Error: If --dm_trans = sut, you must provide a device IP to connect to via the --deviceIP option"
@ -502,18 +369,7 @@ def main(args):
automation.setProduct(options.remoteProductName)
# Set up the defaults and ensure options are set
options = parser.verifyRemoteOptions(options)
if (options == None):
print "ERROR: Invalid options specified, use --help for a list of valid options"
return 1
if not options.ignoreWindowSize:
parts = dm.getInfo('screen')['screen'][0].split()
width = int(parts[0].split(':')[1])
height = int(parts[1].split(':')[1])
if (width < 1050 or height < 1050):
print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (width, height)
return 1
parser.validate_remote(options, automation)
# Check that Firefox is installed
expected = options.app.split('/')[-1]
@ -526,7 +382,7 @@ def main(args):
automation.setRemoteProfile(options.remoteProfile)
automation.setRemoteLog(options.remoteLogFile)
reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY)
options = parser.verifyCommonOptions(options, reftest)
parser.validate(options, reftest)
if mozinfo.info['debug']:
print "changing timeout for remote debug reftests from %s to 600 seconds" % options.timeout
@ -535,17 +391,6 @@ def main(args):
# Hack in a symbolic link for jsreftest
os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest")))
# Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
manifest = args[0]
if os.path.exists(os.path.join(SCRIPT_DIRECTORY, args[0])):
manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + args[0]
elif os.path.exists(args[0]):
manifestPath = os.path.abspath(args[0]).split(SCRIPT_DIRECTORY)[1].strip('/')
manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + manifestPath
else:
print "ERROR: Could not find test manifest '%s'" % manifest
return 1
# Start the webserver
retVal = reftest.startWebServer(options)
if retVal:
@ -561,11 +406,8 @@ def main(args):
# manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list"
retVal = 0
try:
cmdlineArgs = ["-reftest", manifest]
if options.bootstrap:
cmdlineArgs = []
dm.recordLogcat()
retVal = reftest.runTests(manifest, options, cmdlineArgs)
retVal = reftest.runTests(options.tests, options)
except:
print "Automation Error: Exception caught while running tests"
traceback.print_exc()
@ -578,5 +420,5 @@ def main(args):
return retVal
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
sys.exit(main())

334
layout/tools/reftest/runreftest.py
View File

@ -6,9 +6,8 @@
Runs the reftest test harness.
"""
from optparse import OptionParser
from urlparse import urlparse
import collections
import json
import multiprocessing
import os
import re
@ -19,8 +18,9 @@ import sys
import threading
SCRIPT_DIRECTORY = os.path.abspath(
os.path.realpath(os.path.dirname(sys.argv[0])))
sys.path.insert(0, SCRIPT_DIRECTORY)
os.path.realpath(os.path.dirname(__file__)))
if SCRIPT_DIRECTORY not in sys.path:
sys.path.insert(0, SCRIPT_DIRECTORY)
import mozcrash
import mozdebug
@ -32,13 +32,7 @@ import mozrunner
from mozrunner.utils import get_stack_fixer_function, test_environment
from mozscreenshot import printstatus, dump_screen
here = os.path.abspath(os.path.dirname(__file__))
try:
from mozbuild.base import MozbuildObject
build_obj = MozbuildObject.from_environment(cwd=here)
except ImportError:
build_obj = None
import reftestcommandline
# set up logging handler a la automation.py.in for compatability
import logging
@ -132,14 +126,63 @@ class ReftestThread(threading.Thread):
if summaryHeadRegex.search(line) is None:
yield line
class ReftestResolver(object):
def defaultManifest(self, suite):
return {"reftest": "reftest.list",
"crashtest": "crashtests.list",
"jstestbrowser": "jstests.list"}[suite]
def findManifest(self, suite, test_file):
"""Return a tuple of (manifest-path, filter-string) for running test_file.
test_file is a path to a test or a manifest file
"""
if not os.path.isabs(test_file):
test_file = self.absManifestPath(test_file)
if os.path.isdir(test_file):
return os.path.join(test_file, self.defaultManifest(suite)), None
if test_file.endswith('.list'):
return test_file, None
return (self.findManifest(suite, os.path.dirname(test_file))[0],
r".*(?:/|\\)%s$" % os.path.basename(test_file))
def absManifestPath(self, path):
return os.path.abspath(path)
def manifestURL(self, options, path):
return "file://%s" % path
def resolveManifests(self, options, tests):
suite = options.suite
manifests = {}
for testPath in tests:
manifest, filter_str = self.findManifest(suite, testPath)
manifest = self.manifestURL(options, manifest)
if manifest not in manifests:
manifests[manifest] = set()
if filter_str is not None:
manifests[manifest].add(filter_str)
for key in manifests.iterkeys():
if os.path.split(key)[1] != self.defaultManifest(suite):
print >> sys.stderr, "Invalid manifest for suite %s, %s" %(options.suite, key)
sys.exit(1)
manifests[key] = sorted(list(manifests[key]))
return manifests
class RefTest(object):
oldcwd = os.getcwd()
resolver_cls = ReftestResolver
def __init__(self):
self.update_mozinfo()
self.lastTestSeen = 'reftest'
self.haveDumpedScreen = False
self.resolver = self.resolver_cls()
def update_mozinfo(self):
"""walk up directories to find mozinfo.json update the info"""
@ -158,30 +201,15 @@ class RefTest(object):
"Get an absolute path relative to self.oldcwd."
return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
def getManifestPath(self, path):
"Get the path of the manifest, and for remote testing this function is subclassed to point to remote manifest"
path = self.getFullPath(path)
if os.path.isdir(path):
defaultManifestPath = os.path.join(path, 'reftest.list')
if os.path.exists(defaultManifestPath):
path = defaultManifestPath
else:
defaultManifestPath = os.path.join(path, 'crashtests.list')
if os.path.exists(defaultManifestPath):
path = defaultManifestPath
return path
def createReftestProfile(self, options, manifests, server='localhost', port=0,
profile_to_clone=None):
"""Sets up a profile for reftest.
def makeJSString(self, s):
return '"%s"' % re.sub(r'([\\"])', r'\\\1', s)
def createReftestProfile(self, options, manifest, server='localhost', port=0,
special_powers=True, profile_to_clone=None):
"""
Sets up a profile for reftest.
'manifest' is the path to the reftest.list file we want to test with. This is used in
the remote subclass in remotereftest.py so we can write it to a preference for the
bootstrap extension.
"""
:param options: Object containing command line options
:param manifests: Dictionary of the form {manifest_path: [filters]}
:param server: Server name to use for http tests
:param profile_to_clone: Path to a profile to use as the basis for the
test profile"""
locations = mozprofile.permissions.ServerLocations()
locations.add_host(server, scheme='http', port=port)
@ -200,12 +228,10 @@ class RefTest(object):
prefs['reftest.logFile'] = options.logFile
if options.ignoreWindowSize:
prefs['reftest.ignoreWindowSize'] = True
if options.filter:
prefs['reftest.filter'] = options.filter
if options.shuffle:
prefs['reftest.shuffle'] = True
prefs['reftest.focusFilterMode'] = options.focusFilterMode
prefs['reftest.uri'] = "file://%s" % os.path.abspath(manifest)
prefs['reftest.manifests'] = json.dumps(manifests)
# Ensure that telemetry is disabled, so we don't connect to the telemetry
# server in the middle of the tests.
@ -250,13 +276,10 @@ class RefTest(object):
thispref[1].strip())
# install the reftest extension bits into the profile
addons = []
addons.append(os.path.join(SCRIPT_DIRECTORY, "reftest"))
addons = [options.reftestExtensionPath]
# I would prefer to use "--install-extension reftest/specialpowers", but that requires tight coordination with
# release engineering and landing on multiple branches at once.
if special_powers and (manifest.endswith('crashtests.list') or manifest.endswith('jstests.list')):
addons.append(os.path.join(SCRIPT_DIRECTORY, 'specialpowers'))
if options.specialPowersExtensionPath is not None:
addons.append(options.specialPowersExtensionPath)
# SpecialPowers requires insecure automation-only features that we
# put behind a pref.
prefs['security.turn_off_all_security_so_that_viruses_can_take_over_this_computer'] = True
@ -337,7 +360,7 @@ class RefTest(object):
if profileDir:
shutil.rmtree(profileDir, True)
def runTests(self, testPath, options, cmdlineArgs=None):
def runTests(self, tests, options, cmdlineArgs=None):
# Despite our efforts to clean up servers started by this script, in practice
# we still see infrequent cases where a process is orphaned and interferes
# with future tests, typically because the old server is keeping the port in use.
@ -346,8 +369,12 @@ class RefTest(object):
self.killNamedOrphans('ssltunnel')
self.killNamedOrphans('xpcshell')
if not options.runTestsInParallel:
return self.runSerialTests(testPath, options, cmdlineArgs)
manifests = self.resolver.resolveManifests(options, tests)
if options.filter:
manifests[""] = options.filter
if not hasattr(options, "runTestsInParallel") or not options.runTestsInParallel:
return self.runSerialTests(manifests, options, cmdlineArgs)
cpuCount = multiprocessing.cpu_count()
@ -378,7 +405,6 @@ class RefTest(object):
jobArgs.remove("--run-tests-in-parallel")
except:
pass
jobArgs.insert(-1, "--no-run-tests-in-parallel")
jobArgs[0:0] = [sys.executable, "-u"]
threads = [ReftestThread(args) for args in perProcessArgs[1:]]
@ -601,7 +627,7 @@ class RefTest(object):
status = 1
return status
def runSerialTests(self, testPath, options, cmdlineArgs=None):
def runSerialTests(self, manifests, options, cmdlineArgs=None):
debuggerInfo = None
if options.debugger:
debuggerInfo = mozdebug.get_debugger_info(options.debugger, options.debuggerArgs,
@ -609,10 +635,9 @@ class RefTest(object):
profileDir = None
try:
reftestlist = self.getManifestPath(testPath)
if cmdlineArgs == None:
cmdlineArgs = []
profile = self.createReftestProfile(options, reftestlist)
profile = self.createReftestProfile(options, manifests)
profileDir = profile.profile # name makes more sense
# browser environment
@ -660,209 +685,26 @@ class RefTest(object):
continue
class ReftestOptions(OptionParser):
def __init__(self):
OptionParser.__init__(self)
defaults = {}
self.add_option("--xre-path",
action="store", type="string", dest="xrePath",
# individual scripts will set a sane default
default=None,
help="absolute path to directory containing XRE (probably xulrunner)")
self.add_option("--symbols-path",
action="store", type="string", dest="symbolsPath",
default=None,
help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
self.add_option("--debugger",
action="store", dest="debugger",
help="use the given debugger to launch the application")
self.add_option("--debugger-args",
action="store", dest="debuggerArgs",
help="pass the given args to the debugger _before_ "
"the application on the command line")
self.add_option("--debugger-interactive",
action="store_true", dest="debuggerInteractive",
help="prevents the test harness from redirecting "
"stdout and stderr for interactive debuggers")
self.add_option("--appname",
action="store", type="string", dest="app",
help="absolute path to application, overriding default")
# Certain paths do not make sense when we're cross compiling Fennec. This
# logic is cribbed from the example in
# python/mozbuild/mozbuild/mach_commands.py.
defaults['app'] = build_obj.get_binary_path() if \
build_obj and build_obj.substs['MOZ_BUILD_APP'] != 'mobile/android' else None
self.add_option("--extra-profile-file",
action="append", dest="extraProfileFiles",
default=[],
help="copy specified files/dirs to testing profile")
self.add_option("--timeout",
action="store", dest="timeout", type="int",
default=5 * 60, # 5 minutes per bug 479518
help="reftest will timeout in specified number of seconds. [default %default s].")
self.add_option("--leak-threshold",
action="store", type="int", dest="defaultLeakThreshold",
default=0,
help="fail if the number of bytes leaked in default "
"processes through refcounted objects (or bytes "
"in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) "
"is greater than the given number")
self.add_option("--utility-path",
action="store", type="string", dest="utilityPath",
help="absolute path to directory containing utility "
"programs (xpcshell, ssltunnel, certutil)")
defaults["utilityPath"] = build_obj.bindir if \
build_obj and build_obj.substs['MOZ_BUILD_APP'] != 'mobile/android' else None
self.add_option("--total-chunks",
type="int", dest="totalChunks",
help="how many chunks to split the tests up into")
defaults["totalChunks"] = None
self.add_option("--this-chunk",
type="int", dest="thisChunk",
help="which chunk to run between 1 and --total-chunks")
defaults["thisChunk"] = None
self.add_option("--log-file",
action="store", type="string", dest="logFile",
default=None,
help="file to log output to in addition to stdout")
defaults["logFile"] = None
self.add_option("--skip-slow-tests",
dest="skipSlowTests", action="store_true",
help="skip tests marked as slow when running")
defaults["skipSlowTests"] = False
self.add_option("--ignore-window-size",
dest="ignoreWindowSize", action="store_true",
help="ignore the window size, which may cause spurious failures and passes")
defaults["ignoreWindowSize"] = False
self.add_option("--install-extension",
action="append", dest="extensionsToInstall",
help="install the specified extension in the testing profile. "
"The extension file's name should be <id>.xpi where <id> is "
"the extension's id as indicated in its install.rdf. "
"An optional path can be specified too.")
defaults["extensionsToInstall"] = []
self.add_option("--run-tests-in-parallel",
action="store_true", dest="runTestsInParallel",
help="run tests in parallel if possible")
self.add_option("--no-run-tests-in-parallel",
action="store_false", dest="runTestsInParallel",
help="do not run tests in parallel")
defaults["runTestsInParallel"] = False
self.add_option("--setenv",
action="append", type="string",
dest="environment", metavar="NAME=VALUE",
help="sets the given variable in the application's "
"environment")
defaults["environment"] = []
self.add_option("--filter",
action="store", type="string", dest="filter",
help="specifies a regular expression (as could be passed to the JS "
"RegExp constructor) to test against URLs in the reftest manifest; "
"only test items that have a matching test URL will be run.")
defaults["filter"] = None
self.add_option("--shuffle",
action="store_true", dest="shuffle",
help="run reftests in random order")
defaults["shuffle"] = False
self.add_option("--focus-filter-mode",
action="store", type="string", dest="focusFilterMode",
help="filters tests to run by whether they require focus. "
"Valid values are `all', `needs-focus', or `non-needs-focus'. "
"Defaults to `all'.")
defaults["focusFilterMode"] = "all"
self.add_option("--e10s",
action="store_true",
dest="e10s",
help="enables content processes")
defaults["e10s"] = False
self.add_option("--setpref",
action="append", type="string",
default=[],
dest="extraPrefs", metavar="PREF=VALUE",
help="defines an extra user preference")
self.set_defaults(**defaults)
def verifyCommonOptions(self, options, reftest):
if options.totalChunks is not None and options.thisChunk is None:
self.error("thisChunk must be specified when totalChunks is specified")
if options.totalChunks:
if not 1 <= options.thisChunk <= options.totalChunks:
self.error("thisChunk must be between 1 and totalChunks")
if options.logFile:
options.logFile = reftest.getFullPath(options.logFile)
if options.xrePath is not None:
if not os.access(options.xrePath, os.F_OK):
self.error("--xre-path '%s' not found" % options.xrePath)
if not os.path.isdir(options.xrePath):
self.error("--xre-path '%s' is not a directory" %
options.xrePath)
options.xrePath = reftest.getFullPath(options.xrePath)
if options.runTestsInParallel:
if options.logFile is not None:
self.error("cannot specify logfile with parallel tests")
if options.totalChunks is not None and options.thisChunk is None:
self.error("cannot specify thisChunk or totalChunks with parallel tests")
if options.focusFilterMode != "all":
self.error("cannot specify focusFilterMode with parallel tests")
if options.debugger is not None:
self.error("cannot specify a debugger with parallel tests")
options.leakThresholds = {
"default": options.defaultLeakThreshold,
"tab": 5000, # See dependencies of bug 1051230.
}
return options
def run(**kwargs):
# Mach gives us kwargs; this is a way to turn them back into an
# options object
parser = reftestcommandline.DesktopArgumentsParser()
reftest = RefTest()
parser.set_defaults(**kwargs)
options = parser.parse_args(kwargs["tests"])
parser.validate(options, reftest)
return reftest.runTests(options.tests, options)
def main():
parser = ReftestOptions()
parser = reftestcommandline.DesktopArgumentsParser()
reftest = RefTest()
options, args = parser.parse_args()
if len(args) != 1:
print >>sys.stderr, "No reftest.list specified."
sys.exit(1)
options = parser.parse_args()
parser.validate(options, reftest)
options = parser.verifyCommonOptions(options, reftest)
if options.app is None:
parser.error("could not find the application path, --appname must be specified")
sys.exit(reftest.runTests(options.tests, options))
options.app = reftest.getFullPath(options.app)
if not os.path.exists(options.app):
print """Error: Path %(app)s doesn't exist.
Are you executing $objdir/_tests/reftest/runreftest.py?""" \
% {"app": options.app}
sys.exit(1)
if options.xrePath is None:
options.xrePath = os.path.dirname(options.app)
if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2:
options.symbolsPath = reftest.getFullPath(options.symbolsPath)
options.utilityPath = reftest.getFullPath(options.utilityPath)
sys.exit(reftest.runTests(args[0], options))
if __name__ == "__main__":
main()

254
layout/tools/reftest/runreftestb2g.py
View File

@ -10,207 +10,18 @@ import traceback
# We need to know our current directory so that we can serve our test files from it.
here = os.path.abspath(os.path.dirname(__file__))
if here not in sys.path:
sys.path.insert(0, here)
from automation import Automation
from b2gautomation import B2GRemoteAutomation
from b2g_desktop import run_desktop_reftests
from remotereftest import RemoteReftestResolver, ReftestServer
from runreftest import RefTest
from runreftest import ReftestOptions
from remotereftest import ReftestServer
import reftestcommandline
from mozdevice import DeviceManagerADB, DMError
from marionette import Marionette
import moznetwork
class B2GOptions(ReftestOptions):
def __init__(self, **kwargs):
defaults = {}
ReftestOptions.__init__(self)
# This is only used for procName in run_remote_reftests.
defaults["app"] = Automation.DEFAULT_APP
self.add_option("--browser-arg", action="store",
type = "string", dest = "browser_arg",
help = "Optional command-line arg to pass to the browser")
defaults["browser_arg"] = None
self.add_option("--b2gpath", action="store",
type = "string", dest = "b2gPath",
help = "path to B2G repo or qemu dir")
defaults["b2gPath"] = None
self.add_option("--marionette", action="store",
type = "string", dest = "marionette",
help = "host:port to use when connecting to Marionette")
defaults["marionette"] = None
self.add_option("--emulator", action="store",
type="string", dest = "emulator",
help = "Architecture of emulator to use: x86 or arm")
defaults["emulator"] = None
self.add_option("--emulator-res", action="store",
type="string", dest = "emulator_res",
help = "Emulator resolution of the format '<width>x<height>'")
defaults["emulator_res"] = None
self.add_option("--no-window", action="store_true",
dest = "noWindow",
help = "Pass --no-window to the emulator")
defaults["noWindow"] = False
self.add_option("--adbpath", action="store",
type = "string", dest = "adb_path",
help = "path to adb")
defaults["adb_path"] = "adb"
self.add_option("--deviceIP", action="store",
type = "string", dest = "deviceIP",
help = "ip address of remote device to test")
defaults["deviceIP"] = None
self.add_option("--devicePort", action="store",
type = "string", dest = "devicePort",
help = "port of remote device to test")
defaults["devicePort"] = 20701
self.add_option("--remote-logfile", action="store",
type = "string", dest = "remoteLogFile",
help = "Name of log file on the device relative to the device root. PLEASE ONLY USE A FILENAME.")
defaults["remoteLogFile"] = None
self.add_option("--remote-webserver", action = "store",
type = "string", dest = "remoteWebServer",
help = "ip address where the remote web server is hosted at")
defaults["remoteWebServer"] = None
self.add_option("--http-port", action = "store",
type = "string", dest = "httpPort",
help = "ip address where the remote web server is hosted at")
defaults["httpPort"] = None
self.add_option("--ssl-port", action = "store",
type = "string", dest = "sslPort",
help = "ip address where the remote web server is hosted at")
defaults["sslPort"] = None
self.add_option("--pidfile", action = "store",
type = "string", dest = "pidFile",
help = "name of the pidfile to generate")
defaults["pidFile"] = ""
self.add_option("--gecko-path", action="store",
type="string", dest="geckoPath",
help="the path to a gecko distribution that should "
"be installed on the emulator prior to test")
defaults["geckoPath"] = None
self.add_option("--logdir", action="store",
type="string", dest="logdir",
help="directory to store log files")
defaults["logdir"] = None
self.add_option('--busybox', action='store',
type='string', dest='busybox',
help="Path to busybox binary to install on device")
defaults['busybox'] = None
self.add_option("--httpd-path", action = "store",
type = "string", dest = "httpdPath",
help = "path to the httpd.js file")
defaults["httpdPath"] = None
self.add_option("--profile", action="store",
type="string", dest="profile",
help="for desktop testing, the path to the "
"gaia profile to use")
defaults["profile"] = None
self.add_option("--desktop", action="store_true",
dest="desktop",
help="Run the tests on a B2G desktop build")
defaults["desktop"] = False
self.add_option("--mulet", action="store_true",
dest="mulet",
help="Run the tests on a B2G desktop build")
defaults["mulet"] = False
self.add_option("--enable-oop", action="store_true",
dest="oop",
help="Run the tests out of process")
defaults["oop"] = False
defaults["remoteTestRoot"] = None
defaults["logFile"] = "reftest.log"
defaults["autorun"] = True
defaults["closeWhenDone"] = True
defaults["testPath"] = ""
defaults["runTestsInParallel"] = False
self.set_defaults(**defaults)
def verifyRemoteOptions(self, options, auto):
if options.runTestsInParallel:
self.error("Cannot run parallel tests here")
if not options.remoteTestRoot:
options.remoteTestRoot = auto._devicemanager.deviceRoot + "/reftest"
options.remoteProfile = options.remoteTestRoot + "/profile"
productRoot = options.remoteTestRoot + "/" + auto._product
if options.utilityPath is None:
options.utilityPath = productRoot + "/bin"
if options.remoteWebServer == None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
print "ERROR: you must specify a --remote-webserver=<ip address>\n"
return None
options.webServer = options.remoteWebServer
if not options.httpPort:
options.httpPort = auto.DEFAULT_HTTP_PORT
if not options.sslPort:
options.sslPort = auto.DEFAULT_SSL_PORT
if options.geckoPath and not options.emulator:
self.error("You must specify --emulator if you specify --gecko-path")
if options.logdir and not options.emulator:
self.error("You must specify --emulator if you specify --logdir")
#if not options.emulator and not options.deviceIP:
# print "ERROR: you must provide a device IP"
# return None
if options.remoteLogFile == None:
options.remoteLogFile = "reftest.log"
options.localLogName = options.remoteLogFile
options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
# Ensure that the options.logfile (which the base class uses) is set to
# the remote setting when running remote. Also, if the user set the
# log file name there, use that instead of reusing the remotelogfile as above.
if (options.logFile):
# If the user specified a local logfile name use that
options.localLogName = options.logFile
options.logFile = options.remoteLogFile
# Only reset the xrePath if it wasn't provided
if options.xrePath == None:
options.xrePath = options.utilityPath
options.xrePath = os.path.abspath(options.xrePath)
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# httpd-path is specified by standard makefile targets and may be specified
# on the command line to select a particular version of httpd.js. If not
# specified, try to select the one from from the xre bundle, as required in bug 882932.
if not options.httpdPath:
options.httpdPath = os.path.join(options.xrePath, "components")
return options
class ProfileConfigParser(ConfigParser.RawConfigParser):
"""Subclass of RawConfigParser that outputs .ini files in the exact
@ -243,6 +54,7 @@ class B2GRemoteReftest(RefTest):
localProfile = None
remoteApp = ''
profile = None
resolver_cls = RemoteReftestResolver
def __init__(self, automation, devicemanager, options, scriptDir):
RefTest.__init__(self)
@ -418,10 +230,9 @@ class B2GRemoteReftest(RefTest):
pass
def createReftestProfile(self, options, reftestlist):
profile = RefTest.createReftestProfile(self, options, reftestlist,
server=options.remoteWebServer,
special_powers=False)
def createReftestProfile(self, options, manifests):
profile = RefTest.createReftestProfile(self, options, manifests,
server=options.remoteWebServer)
profileDir = profile.profile
prefs = {}
@ -437,7 +248,7 @@ class B2GRemoteReftest(RefTest):
prefs["network.dns.localDomains"] = "app://test-container.gaiamobile.org"
prefs["reftest.browser.iframe.enabled"] = False
prefs["reftest.remote"] = True
prefs["reftest.uri"] = "%s" % reftestlist
# Set a future policy version to avoid the telemetry prompt.
prefs["toolkit.telemetry.prompted"] = 999
prefs["toolkit.telemetry.notifiedOptOut"] = 999
@ -495,9 +306,6 @@ class B2GRemoteReftest(RefTest):
print "Automation Error: Failed to copy extra files to device"
raise
def getManifestPath(self, path):
return path
def environment(self, **kwargs):
return self.automation.environment(**kwargs)
@ -516,7 +324,7 @@ class B2GRemoteReftest(RefTest):
return status
def run_remote_reftests(parser, options, args):
def run_remote_reftests(parser, options):
auto = B2GRemoteAutomation(None, "fennec", context_chrome=True)
# create our Marionette instance
@ -559,11 +367,7 @@ def run_remote_reftests(parser, options, args):
dm = DeviceManagerADB(**kwargs)
auto.setDeviceManager(dm)
options = parser.verifyRemoteOptions(options, auto)
if (options == None):
print "ERROR: Invalid options specified, use --help for a list of valid options"
sys.exit(1)
parser.validate_remote(options, auto)
# TODO fix exception
if not options.ignoreWindowSize:
@ -580,7 +384,7 @@ def run_remote_reftests(parser, options, args):
auto.logFinish = "REFTEST TEST-START | Shutdown"
reftest = B2GRemoteReftest(auto, dm, options, here)
options = parser.verifyCommonOptions(options, reftest)
parser.validate(options, reftest)
logParent = os.path.dirname(options.remoteLogFile)
dm.mkDir(logParent);
@ -590,16 +394,6 @@ def run_remote_reftests(parser, options, args):
# Hack in a symbolic link for jsreftest
os.system("ln -s %s %s" % (os.path.join('..', 'jsreftest'), os.path.join(here, 'jsreftest')))
# Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
manifest = args[0]
if os.path.exists(os.path.join(here, args[0])):
manifest = "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, args[0])
elif os.path.exists(args[0]):
manifestPath = os.path.abspath(args[0]).split(here)[1].strip('/')
manifest = "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, manifestPath)
else:
print "ERROR: Could not find test manifest '%s'" % manifest
return 1
# Start the webserver
retVal = 1
@ -611,11 +405,7 @@ def run_remote_reftests(parser, options, args):
if (dm.processExist(procName)):
dm.killProcess(procName)
cmdlineArgs = ["-reftest", manifest]
if getattr(options, 'bootstrap', False):
cmdlineArgs = []
retVal = reftest.runTests(manifest, options, cmdlineArgs)
retVal = reftest.runTests(options.tests, options)
except:
print "Automation Error: Exception caught while running tests"
traceback.print_exc()
@ -629,13 +419,21 @@ def run_remote_reftests(parser, options, args):
reftest.stopWebServer(options)
return retVal
def main(args=sys.argv[1:]):
parser = B2GOptions()
options, args = parser.parse_args(args)
def run_remote(**kwargs):
# Tests need to be served from a subdirectory of the server. Symlink
# topsrcdir here to get around this.
parser = reftestcommandline.B2GArgumentParser()
parser.set_defaults(**kwargs)
options = parser.parse_args(kwargs["tests"])
return run_remote_reftests(parser, options)
def main():
parser = reftestcommandline.B2GArgumentParser()
options = parser.parse_args()
if options.desktop or options.mulet:
return run_desktop_reftests(parser, options, args)
return run_remote_reftests(parser, options, args)
return run_desktop_reftests(parser, options)
return run_remote_reftests(parser, options)
if __name__ == "__main__":

4
testing/testsuite-targets.mk
View File

@ -174,7 +174,7 @@ REMOTE_REFTEST = rm -f ./$@.log && $(PYTHON) _tests/reftest/remotereftest.py \
--dm_trans=$(DM_TRANS) --ignore-window-size \
--app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \
--httpd-path=_tests/modules \
$(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) '$(1)' | tee ./$@.log
$(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(1) | tee ./$@.log
RUN_REFTEST_B2G = rm -f ./$@.log && $(PYTHON) _tests/reftest/runreftestb2g.py \
--remote-webserver=10.0.2.2 --b2gpath=${B2G_PATH} --adbpath=${ADB_PATH} \
@ -209,7 +209,7 @@ reftest-remote:
echo 'please prepare your host with the environment variable TEST_DEVICE'; \
else \
ln -s $(abspath $(topsrcdir)) _tests/reftest/tests; \
$(call REMOTE_REFTEST,tests/$(TEST_PATH)); \
$(call REMOTE_REFTEST,'tests/$(TEST_PATH)'); \
$(CHECK_TEST_ERROR); \
fi