mirror of
https://gitlab.winehq.org/wine/wine-gecko.git
synced 2024-09-13 09:24:08 -07:00
Bug 1197224 - Add mozharness script to run firefox-media-tests in buildbot, r=jgriffin, DONTBUILD because NPOTB
This commit is contained in:
Maja Frydrychowicz
parent
dde87c210b
commit
c163cee603
@ -0,0 +1,49 @@
|
||||
import os
|
||||
import mozharness
|
||||
|
||||
external_tools_path = os.path.join(
|
||||
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
|
||||
'external_tools',
|
||||
)
|
||||
|
||||
config = {
|
||||
"virtualenv_path": 'venv',
|
||||
"exes": {
|
||||
'python': '/tools/buildbot/bin/python',
|
||||
'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
|
||||
'tooltool.py': "/tools/tooltool.py",
|
||||
'gittool.py': os.path.join(external_tools_path, 'gittool.py'),
|
||||
},
|
||||
|
||||
"find_links": [
|
||||
"http://pypi.pvt.build.mozilla.org/pub",
|
||||
"http://pypi.pub.build.mozilla.org/pub",
|
||||
],
|
||||
"pip_index": False,
|
||||
|
||||
"buildbot_json_path": "buildprops.json",
|
||||
|
||||
"default_actions": [
|
||||
'clobber',
|
||||
'read-buildbot-config',
|
||||
'checkout',
|
||||
'download-and-extract',
|
||||
'create-virtualenv',
|
||||
'install',
|
||||
'run-media-tests',
|
||||
],
|
||||
"default_blob_upload_servers": [
|
||||
"https://blobupload.elasticbeanstalk.com",
|
||||
],
|
||||
"blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
|
||||
"download_minidump_stackwalk": True,
|
||||
"download_symbols": "ondemand",
|
||||
|
||||
"firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
|
||||
"firefox_media_branch": 'master',
|
||||
"firefox_media_rev": '82c45fba24457b5fe447e967bbcaaec5eb14e3ee',
|
||||
"firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
|
||||
"firefox_ui_branch": 'master',
|
||||
"firefox_ui_rev": '6d6d57917f85399e903ac69b7e4297091b2d474c',
|
||||
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
import os
|
||||
import sys
|
||||
import mozharness
|
||||
|
||||
external_tools_path = os.path.join(
|
||||
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
|
||||
'external_tools',
|
||||
)
|
||||
|
||||
config = {
|
||||
"virtualenv_python_dll": 'c:/mozilla-build/python27/python27.dll',
|
||||
"virtualenv_path": 'venv',
|
||||
"exes": {
|
||||
'python': 'c:/mozilla-build/python27/python',
|
||||
'virtualenv': ['c:/mozilla-build/python27/python', 'c:/mozilla-build/buildbotve/virtualenv.py'],
|
||||
'hg': 'c:/mozilla-build/hg/hg',
|
||||
'mozinstall': ['%s/build/venv/scripts/python' % os.getcwd(),
|
||||
'%s/build/venv/scripts/mozinstall-script.py' % os.getcwd()],
|
||||
'tooltool.py': [sys.executable, 'C:/mozilla-build/tooltool.py'],
|
||||
'gittool.py': os.path.join(external_tools_path, 'gittool.py'),
|
||||
|
||||
|
||||
},
|
||||
|
||||
"find_links": [
|
||||
"http://pypi.pvt.build.mozilla.org/pub",
|
||||
"http://pypi.pub.build.mozilla.org/pub",
|
||||
],
|
||||
"pip_index": False,
|
||||
|
||||
"buildbot_json_path": "buildprops.json",
|
||||
|
||||
"default_actions": [
|
||||
'clobber',
|
||||
'read-buildbot-config',
|
||||
'checkout',
|
||||
'download-and-extract',
|
||||
'create-virtualenv',
|
||||
'install',
|
||||
'run-media-tests',
|
||||
],
|
||||
"default_blob_upload_servers": [
|
||||
"https://blobupload.elasticbeanstalk.com",
|
||||
],
|
||||
"blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
|
||||
"in_tree_config": "config/mozharness/marionette.py",
|
||||
"download_minidump_stackwalk": True,
|
||||
"download_symbols": "ondemand",
|
||||
|
||||
"firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
|
||||
"firefox_media_branch": 'master',
|
||||
"firefox_media_rev": '82c45fba24457b5fe447e967bbcaaec5eb14e3ee',
|
||||
"firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
|
||||
"firefox_ui_branch": 'master',
|
||||
"firefox_ui_rev": '6d6d57917f85399e903ac69b7e4297091b2d474c',
|
||||
|
||||
}
|
||||
@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python
|
||||
# ***** BEGIN LICENSE BLOCK *****
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
# ***** BEGIN LICENSE BLOCK *****
|
||||
"""firefox_media_tests.py
|
||||
|
||||
Author: Maja Frydrychowicz
|
||||
"""
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
|
||||
from mozharness.base.log import ERROR, WARNING
|
||||
from mozharness.base.script import PreScriptAction
|
||||
from mozharness.mozilla.testing.testbase import (TestingMixin,
|
||||
testing_config_options)
|
||||
from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
|
||||
from mozharness.mozilla.vcstools import VCSToolsScript
|
||||
|
||||
BUSTED = 'busted'
|
||||
TESTFAILED = 'testfailed'
|
||||
UNKNOWN = 'unknown'
|
||||
EXCEPTION = 'exception'
|
||||
SUCCESS = 'success'
|
||||
|
||||
media_test_config_options = [
|
||||
[["--media-urls"],
|
||||
{"action": "store",
|
||||
"dest": "media_urls",
|
||||
"help": "Path to ini file that lists media urls for tests.",
|
||||
}],
|
||||
[["--profile"],
|
||||
{"action": "store",
|
||||
"dest": "profile",
|
||||
"default": None,
|
||||
"help": "Path to FF profile that should be used by Marionette",
|
||||
}],
|
||||
[["--test-timeout"],
|
||||
{"action": "store",
|
||||
"dest": "test_timeout",
|
||||
"default": 10000,
|
||||
"help": ("Number of seconds without output before"
|
||||
"firefox-media-tests is killed."
|
||||
"Set this based on expected time for all media to play."),
|
||||
}],
|
||||
[["--tests"],
|
||||
{"action": "store",
|
||||
"dest": "tests",
|
||||
"default": None,
|
||||
"help": ("Test(s) to run. Path to test_*.py or "
|
||||
"test manifest (*.ini)"),
|
||||
}],
|
||||
[["--e10s"],
|
||||
{"dest": "e10s",
|
||||
"action": "store_true",
|
||||
"default": False,
|
||||
"help": "Enable e10s when running marionette tests."
|
||||
}],
|
||||
[['--firefox-media-repo'], {
|
||||
'dest': 'firefox_media_repo',
|
||||
'default': 'https://github.com/mjzffr/firefox-media-tests.git',
|
||||
'help': 'which firefox_media_tests repo to use',
|
||||
}],
|
||||
[['--firefox-media-branch'], {
|
||||
'dest': 'firefox_media_branch',
|
||||
'default': 'master',
|
||||
'help': 'which branch to use for firefox_media_tests',
|
||||
}],
|
||||
[['--firefox-media-rev'], {
|
||||
'dest': 'firefox_media_rev',
|
||||
'help': 'which firefox_media_tests revision to use',
|
||||
}],
|
||||
[['--firefox-ui-repo'], {
|
||||
'dest': 'firefox_ui_repo',
|
||||
'default': 'https://github.com/mozilla/firefox-ui-tests.git',
|
||||
'help': 'which firefox_ui_tests repo to use',
|
||||
}],
|
||||
[['--firefox-ui-branch'], {
|
||||
'dest': 'firefox_ui_branch',
|
||||
'default': 'master',
|
||||
'help': 'which branch to use for firefox_ui_tests',
|
||||
}],
|
||||
[['--firefox-ui-rev'], {
|
||||
'dest': 'firefox_ui_rev',
|
||||
'help': 'which firefox_ui_tests revision to use',
|
||||
}],
|
||||
] + (copy.deepcopy(testing_config_options))
|
||||
|
||||
|
||||
class JobResultParser(TestSummaryOutputParserHelper):
|
||||
""" Parses test output to determine overall result."""
|
||||
def __init__(self, **kwargs):
|
||||
super(JobResultParser, self).__init__(**kwargs)
|
||||
self.return_code = 0
|
||||
# External-resource errors that should not count as test failures
|
||||
self.exception_re = re.compile(r'^TEST-UNEXPECTED-ERROR.*'
|
||||
r'TimeoutException: Error loading page,'
|
||||
r' timed out')
|
||||
self.exceptions = []
|
||||
|
||||
def parse_single_line(self, line):
|
||||
super(JobResultParser, self).parse_single_line(line)
|
||||
if self.exception_re.match(line):
|
||||
self.exceptions.append(line)
|
||||
|
||||
@property
|
||||
def status(self):
|
||||
status = UNKNOWN
|
||||
if self.passed and self.failed == 0:
|
||||
status = SUCCESS
|
||||
elif self.exceptions:
|
||||
status = EXCEPTION
|
||||
elif self.failed:
|
||||
status = TESTFAILED
|
||||
elif self.return_code:
|
||||
status = BUSTED
|
||||
return status
|
||||
|
||||
|
||||
class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
|
||||
job_result_parser = None
|
||||
|
||||
error_list = [
|
||||
{'substr': 'FAILED (errors=', 'level': WARNING},
|
||||
{'substr': r'''Could not successfully complete transport of message to Gecko, socket closed''', 'level': ERROR},
|
||||
{'substr': r'''Connection to Marionette server is lost. Check gecko''', 'level': ERROR},
|
||||
{'substr': 'Timeout waiting for marionette on port', 'level': ERROR},
|
||||
{'regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH|CRASH|ERROR|FAIL)'''), 'level': ERROR},
|
||||
{'regex': re.compile(r'''(\b\w*Exception)'''), 'level': ERROR},
|
||||
{'regex': re.compile(r'''(\b\w*Error)'''), 'level': ERROR},
|
||||
]
|
||||
|
||||
def __init__(self, config_options=None, all_actions=None,
|
||||
default_actions=None, **kwargs):
|
||||
self.config_options = media_test_config_options + (config_options or [])
|
||||
actions = [
|
||||
'clobber',
|
||||
'checkout',
|
||||
'create-virtualenv',
|
||||
'run-media-tests',
|
||||
]
|
||||
super(FirefoxMediaTestsBase, self).__init__(
|
||||
config_options=self.config_options,
|
||||
all_actions=all_actions or actions,
|
||||
default_actions=default_actions or actions,
|
||||
**kwargs
|
||||
)
|
||||
c = self.config
|
||||
self.media_urls = c.get('media_urls')
|
||||
self.profile = c.get('profile')
|
||||
self.test_timeout = int(c.get('test_timeout'))
|
||||
self.tests = c.get('tests')
|
||||
self.e10s = c.get('e10s')
|
||||
|
||||
@PreScriptAction('create-virtualenv')
|
||||
def _pre_create_virtualenv(self, action):
|
||||
dirs = self.query_abs_dirs()
|
||||
# cwd is $workspace/build
|
||||
self.register_virtualenv_module(name='firefox-ui-tests',
|
||||
url=dirs['firefox_ui_dir'],
|
||||
method='pip',
|
||||
editable='true')
|
||||
self.register_virtualenv_module(name='firefox-media-tests',
|
||||
url=dirs['firefox_media_dir'],
|
||||
method='pip',
|
||||
editable='true')
|
||||
|
||||
def query_abs_dirs(self):
|
||||
if self.abs_dirs:
|
||||
return self.abs_dirs
|
||||
abs_dirs = super(FirefoxMediaTestsBase, self).query_abs_dirs()
|
||||
dirs = {
|
||||
'firefox_media_dir': os.path.join(abs_dirs['abs_work_dir'],
|
||||
'firefox-media-tests')
|
||||
}
|
||||
dirs['firefox_ui_dir'] = os.path.join(dirs['firefox_media_dir'],
|
||||
'firefox-ui-tests')
|
||||
abs_dirs.update(dirs)
|
||||
self.abs_dirs = abs_dirs
|
||||
return self.abs_dirs
|
||||
|
||||
@PreScriptAction('checkout')
|
||||
def _pre_checkout(self, action):
|
||||
super(FirefoxMediaTestsBase, self)._pre_checkout(action)
|
||||
c = self.config
|
||||
dirs = self.query_abs_dirs()
|
||||
self.firefox_media_vc = {
|
||||
'branch': c['firefox_media_branch'],
|
||||
'repo': c['firefox_media_repo'],
|
||||
'revision': c['firefox_media_rev'],
|
||||
'dest': dirs['firefox_media_dir'],
|
||||
}
|
||||
self.firefox_ui_vc = {
|
||||
'branch': c['firefox_ui_branch'],
|
||||
'repo': c['firefox_ui_repo'],
|
||||
'revision': c['firefox_ui_rev'],
|
||||
'dest': dirs['firefox_ui_dir']
|
||||
}
|
||||
|
||||
def checkout(self):
|
||||
revision = self.vcs_checkout(vcs='gittool', **self.firefox_media_vc)
|
||||
if revision:
|
||||
self.vcs_checkout(vcs='gittool', **self.firefox_ui_vc)
|
||||
|
||||
def _query_cmd(self):
|
||||
""" Determine how to call firefox-media-tests """
|
||||
if not self.binary_path:
|
||||
self.fatal("Binary path could not be determined. "
|
||||
"Should be set by default during 'install' action.")
|
||||
cmd = ['firefox-media-tests']
|
||||
cmd += ['--binary', self.binary_path]
|
||||
if self.symbols_path:
|
||||
cmd += ['--symbols-path', self.symbols_path]
|
||||
if self.media_urls:
|
||||
cmd += ['--urls', self.media_urls]
|
||||
if self.profile:
|
||||
cmd += ['--profile', self.profile]
|
||||
if self.tests:
|
||||
cmd.append(self.tests)
|
||||
if self.e10s:
|
||||
cmd.append('--e10s')
|
||||
return cmd
|
||||
|
||||
def run_media_tests(self):
|
||||
cmd = self._query_cmd()
|
||||
self.job_result_parser = JobResultParser(
|
||||
config=self.config,
|
||||
log_obj=self.log_obj,
|
||||
error_list=self.error_list
|
||||
)
|
||||
return_code = self.run_command(
|
||||
cmd,
|
||||
output_timeout=self.test_timeout,
|
||||
output_parser=self.job_result_parser
|
||||
)
|
||||
self.job_result_parser.return_code = return_code
|
||||
return self.job_result_parser.status
|
||||
142
testing/mozharness/scripts/firefox_media_tests_buildbot.py
Normal file
142
testing/mozharness/scripts/firefox_media_tests_buildbot.py
Normal file
@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python
|
||||
# ***** BEGIN LICENSE BLOCK *****
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
# ***** BEGIN LICENSE BLOCK *****
|
||||
"""firefox_media_tests_buildbot.py
|
||||
|
||||
Author: Maja Frydrychowicz
|
||||
"""
|
||||
import copy
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(1, os.path.dirname(sys.path[0]))
|
||||
|
||||
from mozharness.base.log import ERROR, DEBUG, INFO
|
||||
from mozharness.base.script import PreScriptAction, PostScriptAction
|
||||
from mozharness.mozilla.blob_upload import (
|
||||
BlobUploadMixin,
|
||||
blobupload_config_options
|
||||
)
|
||||
from mozharness.mozilla.buildbot import (
|
||||
TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
|
||||
)
|
||||
from mozharness.mozilla.testing.firefox_media_tests import (
|
||||
FirefoxMediaTestsBase, BUSTED, TESTFAILED, UNKNOWN, EXCEPTION, SUCCESS
|
||||
)
|
||||
|
||||
|
||||
class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
|
||||
|
||||
def __init__(self):
|
||||
config_options = copy.deepcopy(blobupload_config_options)
|
||||
super(FirefoxMediaTestsBuildbot, self).__init__(
|
||||
config_options=config_options,
|
||||
all_actions=['clobber',
|
||||
'read-buildbot-config',
|
||||
'checkout',
|
||||
'download-and-extract',
|
||||
'create-virtualenv',
|
||||
'install',
|
||||
'run-media-tests',
|
||||
],
|
||||
)
|
||||
c = self.config
|
||||
self.installer_url = c.get('installer_url')
|
||||
self.installer_path = c.get('installer_path')
|
||||
self.binary_path = c.get('binary_path')
|
||||
self.test_packages_url = c.get('test_packages_url')
|
||||
|
||||
@PreScriptAction('create-virtualenv')
|
||||
def _pre_create_virtualenv(self, action):
|
||||
dirs = self.query_abs_dirs()
|
||||
requirements = os.path.join(dirs['abs_test_install_dir'],
|
||||
'config',
|
||||
'marionette_requirements.txt')
|
||||
if os.access(requirements, os.F_OK):
|
||||
self.register_virtualenv_module(requirements=[requirements],
|
||||
two_pass=True)
|
||||
super(FirefoxMediaTestsBuildbot, self)._pre_create_virtualenv(action)
|
||||
|
||||
def query_abs_dirs(self):
|
||||
if self.abs_dirs:
|
||||
return self.abs_dirs
|
||||
dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
|
||||
dirs['abs_blob_upload_dir'] = os.path.join(dirs['abs_work_dir'],
|
||||
'blobber_upload_dir')
|
||||
dirs['abs_test_install_dir'] = os.path.join(dirs['abs_work_dir'],
|
||||
'tests')
|
||||
self.abs_dirs = dirs
|
||||
return self.abs_dirs
|
||||
|
||||
def _query_cmd(self):
|
||||
""" Determine how to call firefox-media-tests """
|
||||
cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
|
||||
# configure logging
|
||||
dirs = self.query_abs_dirs()
|
||||
blob_upload_dir = dirs.get('abs_blob_upload_dir')
|
||||
cmd += ['--gecko-log', os.path.join(blob_upload_dir,
|
||||
'gecko.log')]
|
||||
cmd += ['--log-html', os.path.join(blob_upload_dir,
|
||||
'media_tests.html')]
|
||||
cmd += ['--log-mach', os.path.join(blob_upload_dir,
|
||||
'media_tests_mach.log')]
|
||||
return cmd
|
||||
|
||||
def run_media_tests(self):
|
||||
status = super(FirefoxMediaTestsBuildbot, self).run_media_tests()
|
||||
if status == SUCCESS:
|
||||
tbpl_status = TBPL_SUCCESS
|
||||
else:
|
||||
tbpl_status = TBPL_FAILURE
|
||||
if status == TESTFAILED:
|
||||
tbpl_status = TBPL_WARNING
|
||||
self.buildbot_status(tbpl_status)
|
||||
|
||||
@PostScriptAction('run-media-tests')
|
||||
def _collect_uploads(self, action, success=None):
|
||||
""" Copy extra (log) files to blob upload dir. """
|
||||
dirs = self.query_abs_dirs()
|
||||
log_dir = dirs.get('abs_log_dir')
|
||||
blob_upload_dir = dirs.get('abs_blob_upload_dir')
|
||||
if not log_dir or not blob_upload_dir:
|
||||
return
|
||||
self.mkdir_p(blob_upload_dir)
|
||||
# Move firefox-media-test screenshots into log_dir
|
||||
screenshots_dir = os.path.join(dirs['base_work_dir'],
|
||||
'screenshots')
|
||||
log_screenshots_dir = os.path.join(log_dir, 'screenshots')
|
||||
if os.access(log_screenshots_dir, os.F_OK):
|
||||
self.rmtree(log_screenshots_dir)
|
||||
if os.access(screenshots_dir, os.F_OK):
|
||||
self.move(screenshots_dir, log_screenshots_dir)
|
||||
|
||||
# logs to upload: broadest level (info), error, screenshots
|
||||
uploads = glob.glob(os.path.join(log_screenshots_dir, '*'))
|
||||
log_files = self.log_obj.log_files
|
||||
log_level = self.log_obj.log_level
|
||||
|
||||
def append_path(filename, dir=log_dir):
|
||||
if filename:
|
||||
uploads.append(os.path.join(dir, filename))
|
||||
|
||||
append_path(log_files.get(ERROR))
|
||||
# never upload debug logs
|
||||
if log_level == DEBUG:
|
||||
append_path(log_files.get(INFO))
|
||||
else:
|
||||
append_path(log_files.get(log_level))
|
||||
# in case of SimpleFileLogger
|
||||
append_path(log_files.get('default'))
|
||||
for f in uploads:
|
||||
if os.access(f, os.F_OK):
|
||||
dest = os.path.join(blob_upload_dir, os.path.basename(f))
|
||||
self.copyfile(f, dest)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
media_test = FirefoxMediaTestsBuildbot()
|
||||
media_test.run_and_exit()
|
||||
Loading…
Reference in New Issue
Block a user