mirror of
https://gitlab.winehq.org/wine/wine-gecko.git
synced 2024-09-13 09:24:08 -07:00
9118d348c5
This adds support for web-platform-tests to mach try. It changes the implementation so that instead of passing paths to manifests, the user passes arbitary paths in the source tree, and tests under that path are run, with test discovery mainly left to the harness.
412 lines
17 KiB
Python
Executable File
412 lines
17 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# ***** BEGIN LICENSE BLOCK *****
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
# ***** END LICENSE BLOCK *****
|
|
|
|
import copy
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
# load modules from parent dir
|
|
sys.path.insert(1, os.path.dirname(sys.path[0]))
|
|
|
|
from mozharness.base.errors import BaseErrorList, TarErrorList
|
|
from mozharness.base.log import ERROR, WARNING
|
|
from mozharness.base.script import (
|
|
BaseScript,
|
|
PreScriptAction,
|
|
)
|
|
from mozharness.base.vcs.vcsbase import VCSMixin
|
|
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
|
|
from mozharness.mozilla.testing.errors import LogcatErrorList
|
|
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
|
|
from mozharness.mozilla.buildbot import TBPL_SUCCESS
|
|
|
|
|
|
class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
|
|
test_suites = ('jsreftest', 'reftest', 'mochitest', 'mochitest-chrome', 'xpcshell', 'crashtest', 'cppunittest')
|
|
config_options = [[
|
|
["--type"],
|
|
{"action": "store",
|
|
"dest": "test_type",
|
|
"default": "browser",
|
|
"help": "The type of tests to run",
|
|
}
|
|
], [
|
|
["--busybox-url"],
|
|
{"action": "store",
|
|
"dest": "busybox_url",
|
|
"default": None,
|
|
"help": "URL to the busybox binary",
|
|
}
|
|
], [
|
|
["--emulator-url"],
|
|
{"action": "store",
|
|
"dest": "emulator_url",
|
|
"default": None,
|
|
"help": "URL to the emulator zip",
|
|
}
|
|
], [
|
|
["--xre-url"],
|
|
{"action": "store",
|
|
"dest": "xre_url",
|
|
"default": None,
|
|
"help": "URL to the desktop xre zip",
|
|
}
|
|
], [
|
|
["--gecko-url"],
|
|
{"action": "store",
|
|
"dest": "gecko_url",
|
|
"default": None,
|
|
"help": "URL to the gecko build injected into the emulator",
|
|
}
|
|
], [
|
|
["--test-manifest"],
|
|
{"action": "store",
|
|
"dest": "test_manifest",
|
|
"default": None,
|
|
"help": "Path to test manifest to run",
|
|
}
|
|
], [
|
|
["--test-suite"],
|
|
{"action": "store",
|
|
"dest": "test_suite",
|
|
"type": "choice",
|
|
"choices": test_suites,
|
|
"help": "Which test suite to run",
|
|
}
|
|
], [
|
|
["--adb-path"],
|
|
{"action": "store",
|
|
"dest": "adb_path",
|
|
"default": None,
|
|
"help": "Path to adb",
|
|
}
|
|
], [
|
|
["--total-chunks"],
|
|
{"action": "store",
|
|
"dest": "total_chunks",
|
|
"help": "Number of total chunks",
|
|
}
|
|
], [
|
|
["--this-chunk"],
|
|
{"action": "store",
|
|
"dest": "this_chunk",
|
|
"help": "Number of this chunk",
|
|
}
|
|
], [
|
|
["--test-path"],
|
|
{"action": "store",
|
|
"dest": "test_path",
|
|
"help": "Path of tests to run",
|
|
}
|
|
]] + copy.deepcopy(testing_config_options) \
|
|
+ copy.deepcopy(blobupload_config_options)
|
|
|
|
error_list = [
|
|
{'substr': 'FAILED (errors=', 'level': ERROR},
|
|
{'substr': r'''Could not successfully complete transport of message to Gecko, socket closed''', 'level': ERROR},
|
|
{'substr': r'''Could not communicate with Marionette server. Is the Gecko process still running''', 'level': ERROR},
|
|
{'substr': r'''Connection to Marionette server is lost. Check gecko''', 'level': ERROR},
|
|
{'substr': 'Timeout waiting for marionette on port', 'level': ERROR},
|
|
{'regex': re.compile(r'''(Timeout|NoSuchAttribute|Javascript|NoSuchElement|XPathLookup|NoSuchWindow|StaleElement|ScriptTimeout|ElementNotVisible|NoSuchFrame|InvalidElementState|NoAlertPresent|InvalidCookieDomain|UnableToSetCookie|InvalidSelector|MoveTargetOutOfBounds)Exception'''), 'level': ERROR},
|
|
]
|
|
|
|
def __init__(self, require_config_file=False):
|
|
super(B2GEmulatorTest, self).__init__(
|
|
config_options=self.config_options,
|
|
all_actions=['clobber',
|
|
'read-buildbot-config',
|
|
'download-and-extract',
|
|
'create-virtualenv',
|
|
'install',
|
|
'run-tests'],
|
|
default_actions=['clobber',
|
|
'download-and-extract',
|
|
'create-virtualenv',
|
|
'install',
|
|
'run-tests'],
|
|
require_config_file=require_config_file,
|
|
config={
|
|
'require_test_zip': True,
|
|
# This is a special IP that has meaning to the emulator
|
|
'remote_webserver': '10.0.2.2',
|
|
}
|
|
)
|
|
|
|
# these are necessary since self.config is read only
|
|
c = self.config
|
|
self.adb_path = c.get('adb_path', self._query_adb())
|
|
self.installer_url = c.get('installer_url')
|
|
self.installer_path = c.get('installer_path')
|
|
self.test_url = c.get('test_url')
|
|
self.test_packages_url = c.get('test_packages_url')
|
|
self.test_manifest = c.get('test_manifest')
|
|
self.busybox_path = None
|
|
|
|
# TODO detect required config items and fail if not set
|
|
|
|
def query_abs_dirs(self):
|
|
if self.abs_dirs:
|
|
return self.abs_dirs
|
|
abs_dirs = super(B2GEmulatorTest, self).query_abs_dirs()
|
|
dirs = {}
|
|
dirs['abs_test_install_dir'] = os.path.join(
|
|
abs_dirs['abs_work_dir'], 'tests')
|
|
dirs['abs_xre_dir'] = os.path.join(
|
|
abs_dirs['abs_work_dir'], 'xre')
|
|
dirs['abs_emulator_dir'] = os.path.join(
|
|
abs_dirs['abs_work_dir'], 'emulator')
|
|
dirs['abs_blob_upload_dir'] = os.path.join(
|
|
abs_dirs['abs_work_dir'], 'blobber_upload_dir')
|
|
dirs['abs_b2g-distro_dir'] = os.path.join(
|
|
dirs['abs_emulator_dir'], 'b2g-distro')
|
|
dirs['abs_mochitest_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'mochitest')
|
|
dirs['abs_mochitest-chrome_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'mochitest')
|
|
dirs['abs_certs_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'certs')
|
|
dirs['abs_modules_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'modules')
|
|
dirs['abs_reftest_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'reftest')
|
|
dirs['abs_crashtest_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'reftest')
|
|
dirs['abs_jsreftest_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'reftest')
|
|
dirs['abs_xpcshell_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'xpcshell')
|
|
dirs['abs_cppunittest_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'cppunittest')
|
|
for key in dirs.keys():
|
|
if key not in abs_dirs:
|
|
abs_dirs[key] = dirs[key]
|
|
self.abs_dirs = abs_dirs
|
|
return self.abs_dirs
|
|
|
|
def download_and_extract(self):
|
|
target_suite = self.config['test_suite']
|
|
super(B2GEmulatorTest, self).download_and_extract(suite_categories=[target_suite])
|
|
dirs = self.query_abs_dirs()
|
|
|
|
self.mkdir_p(dirs['abs_emulator_dir'])
|
|
tar = self.query_exe('tar', return_type='list')
|
|
self.run_command(tar + ['zxf', self.installer_path],
|
|
cwd=dirs['abs_emulator_dir'],
|
|
error_list=TarErrorList,
|
|
halt_on_failure=True, fatal_exit_code=3)
|
|
|
|
self.mkdir_p(dirs['abs_xre_dir'])
|
|
self._download_unzip(self.config['xre_url'],
|
|
dirs['abs_xre_dir'])
|
|
|
|
if self.config.get('busybox_url'):
|
|
self.download_file(self.config['busybox_url'],
|
|
file_name='busybox',
|
|
parent_dir=dirs['abs_work_dir'])
|
|
self.busybox_path = os.path.join(dirs['abs_work_dir'], 'busybox')
|
|
|
|
@PreScriptAction('create-virtualenv')
|
|
def _pre_create_virtualenv(self, action):
|
|
dirs = self.query_abs_dirs()
|
|
requirements = os.path.join(dirs['abs_test_install_dir'],
|
|
'config',
|
|
'marionette_requirements.txt')
|
|
if os.path.isfile(requirements):
|
|
self.register_virtualenv_module(requirements=[requirements],
|
|
two_pass=True)
|
|
return
|
|
|
|
# XXX Bug 879765: Dependent modules need to be listed before parent
|
|
# modules, otherwise they will get installed from the pypi server.
|
|
# XXX Bug 908356: This block can be removed as soon as the
|
|
# in-tree requirements files propagate to all active trees.
|
|
mozbase_dir = os.path.join('tests', 'mozbase')
|
|
self.register_virtualenv_module(
|
|
'manifestparser',
|
|
url=os.path.join(mozbase_dir, 'manifestdestiny')
|
|
)
|
|
|
|
for m in ('mozfile', 'mozlog', 'mozinfo', 'moznetwork', 'mozhttpd',
|
|
'mozcrash', 'mozinstall', 'mozdevice', 'mozprofile', 'mozprocess',
|
|
'mozrunner'):
|
|
self.register_virtualenv_module(
|
|
m, url=os.path.join(mozbase_dir, m))
|
|
|
|
self.register_virtualenv_module(
|
|
'marionette', url=os.path.join('tests', 'marionette'))
|
|
|
|
def _query_abs_base_cmd(self, suite):
|
|
dirs = self.query_abs_dirs()
|
|
cmd = [self.query_python_path('python')]
|
|
cmd.append(self.config['run_file_names'][suite])
|
|
|
|
raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
|
|
'%s_raw.log' % suite)
|
|
error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
|
|
'%s_errorsummary.log' % suite)
|
|
emulator_type = 'x86' if os.path.isdir(os.path.join(dirs['abs_b2g-distro_dir'],
|
|
'out', 'target', 'product', 'generic_x86')) else 'arm'
|
|
self.info("The emulator type: %s" % emulator_type)
|
|
|
|
str_format_values = {
|
|
'adbpath': self.adb_path,
|
|
'b2gpath': dirs['abs_b2g-distro_dir'],
|
|
'emulator': emulator_type,
|
|
'logcat_dir': dirs['abs_work_dir'],
|
|
'modules_dir': dirs['abs_modules_dir'],
|
|
'remote_webserver': self.config['remote_webserver'],
|
|
'xre_path': os.path.join(dirs['abs_xre_dir'], 'bin'),
|
|
'utility_path': os.path.join(dirs['abs_test_install_dir'], 'bin'),
|
|
'symbols_path': self.symbols_path,
|
|
'busybox': self.busybox_path,
|
|
'total_chunks': self.config.get('total_chunks'),
|
|
'this_chunk': self.config.get('this_chunk'),
|
|
'test_path': self.config.get('test_path'),
|
|
'certificate_path': dirs['abs_certs_dir'],
|
|
'raw_log_file': raw_log_file,
|
|
'error_summary_file': error_summary_file,
|
|
}
|
|
|
|
if suite not in self.config["suite_definitions"]:
|
|
self.fatal("'%s' not defined in the config!" % suite)
|
|
|
|
try_options, try_tests = self.try_args(suite)
|
|
|
|
options = self.query_options(self.config["suite_definitions"][suite]["options"],
|
|
try_options,
|
|
str_format_values=str_format_values)
|
|
cmd.extend(opt for opt in options if not opt.endswith('None'))
|
|
|
|
tests = self.query_tests_args(self.config["suite_definitions"][suite].get("tests"),
|
|
try_tests,
|
|
str_format_values=str_format_values)
|
|
cmd.extend(opt for opt in tests if not opt.endswith('None'))
|
|
|
|
return cmd
|
|
|
|
def _query_adb(self):
|
|
return self.which('adb') or \
|
|
os.getenv('ADB_PATH') or \
|
|
os.path.join(self.query_abs_dirs()['abs_b2g-distro_dir'],
|
|
'out', 'host', 'linux-x86', 'bin', 'adb')
|
|
|
|
def preflight_run_tests(self):
|
|
super(B2GEmulatorTest, self).preflight_run_tests()
|
|
suite = self.config['test_suite']
|
|
# set default test manifest by suite if none specified
|
|
if not self.test_manifest:
|
|
if suite == 'reftest':
|
|
self.test_manifest = os.path.join('tests', 'layout',
|
|
'reftests', 'reftest.list')
|
|
elif suite == 'xpcshell':
|
|
self.test_manifest = os.path.join('tests', 'xpcshell_b2g.ini')
|
|
elif suite == 'crashtest':
|
|
self.test_manifest = os.path.join('tests', 'testing',
|
|
'crashtest', 'crashtests.list')
|
|
elif suite == 'jsreftest':
|
|
self.test_manifest = os.path.join('jsreftest', 'tests',
|
|
'jstests.list')
|
|
|
|
if not os.path.isfile(self.adb_path):
|
|
self.fatal("The adb binary '%s' is not a valid file!" % self.adb_path)
|
|
|
|
def install(self):
|
|
# The emulator was extracted during the download_and_extract step;
|
|
# there's no separate binary to install. We call pass to prevent
|
|
# the base implementation from running here.
|
|
pass
|
|
|
|
def _get_success_codes(self, suite_name):
|
|
success_codes = None
|
|
if suite_name == 'xpcshell':
|
|
# bug 773703
|
|
success_codes = [0, 1]
|
|
return success_codes
|
|
|
|
def run_tests(self):
|
|
"""
|
|
Run the tests
|
|
"""
|
|
dirs = self.query_abs_dirs()
|
|
|
|
error_list = self.error_list
|
|
error_list.extend(BaseErrorList)
|
|
|
|
suite = self.config['test_suite']
|
|
if suite not in self.test_suites:
|
|
self.fatal("Don't know how to run --test-suite '%s'!" % suite)
|
|
|
|
cmd = self._query_abs_base_cmd(suite)
|
|
cwd = dirs['abs_%s_dir' % suite]
|
|
|
|
# TODO we probably have to move some of the code in
|
|
# scripts/desktop_unittest.py and scripts/marionette.py to
|
|
# mozharness.mozilla.testing.unittest so we can share it.
|
|
# In the short term, I'm ok with some duplication of code if it
|
|
# expedites things; please file bugs to merge if that happens.
|
|
|
|
suite_name = [x for x in self.test_suites if x in self.config['test_suite']][0]
|
|
if self.config.get('this_chunk'):
|
|
suite = '%s-%s' % (suite_name, self.config['this_chunk'])
|
|
else:
|
|
suite = suite_name
|
|
|
|
env = {}
|
|
if self.query_minidump_stackwalk():
|
|
env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
|
|
env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir']
|
|
if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
|
|
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
|
|
env = self.query_env(partial_env=env)
|
|
|
|
success_codes = self._get_success_codes(suite_name)
|
|
parser = self.get_test_output_parser(suite_name,
|
|
config=self.config,
|
|
log_obj=self.log_obj,
|
|
error_list=error_list)
|
|
return_code = self.run_command(cmd, cwd=cwd, env=env,
|
|
output_timeout=1000,
|
|
output_parser=parser,
|
|
success_codes=success_codes)
|
|
|
|
logcat = os.path.join(dirs['abs_work_dir'], 'emulator-5554.log')
|
|
|
|
qemu = os.path.join(dirs['abs_work_dir'], 'qemu.log')
|
|
if os.path.isfile(qemu):
|
|
self.copyfile(qemu, os.path.join(env['MOZ_UPLOAD_DIR'],
|
|
os.path.basename(qemu)))
|
|
|
|
tbpl_status, log_level = parser.evaluate_parser(return_code,
|
|
success_codes=success_codes)
|
|
|
|
if os.path.isfile(logcat):
|
|
if tbpl_status != TBPL_SUCCESS:
|
|
# On failure, dump logcat, check if the emulator is still
|
|
# running, and if it is still accessible via adb.
|
|
self.info('dumping logcat')
|
|
self.run_command(['cat', logcat], error_list=LogcatErrorList)
|
|
|
|
self.run_command(['ps', '-C', 'emulator'])
|
|
self.run_command([self.adb_path, 'devices'])
|
|
|
|
# upload logcat to blobber
|
|
self.copyfile(logcat, os.path.join(env['MOZ_UPLOAD_DIR'],
|
|
os.path.basename(logcat)))
|
|
else:
|
|
self.info('no logcat file found')
|
|
|
|
parser.append_tinderboxprint_line(suite_name)
|
|
|
|
self.buildbot_status(tbpl_status, level=log_level)
|
|
self.log("The %s suite: %s ran with return status: %s" %
|
|
(suite_name, suite, tbpl_status), level=log_level)
|
|
|
|
if __name__ == '__main__':
|
|
emulatorTest = B2GEmulatorTest()
|
|
emulatorTest.run_and_exit()
|