wine/gecko
0
0
Fork 0
mirror of https://gitlab.winehq.org/wine/wine-gecko.git synced 2024-09-13 09:24:08 -07:00
Code Issues Packages Projects Releases Wiki Activity

Bug 1230279 - Develop new mozharness script for media-tests for Jenkins; r=maja_zf r=whimboo

Browse Source
This commit is contained in:
Syd Polk 2016-01-11 15:52:25 -06:00
parent ae78fa8cd9
commit 3d3918c956
6 changed files with 237 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
testing/mozharness/configs/mediatests/buildbot_posix_config.py
View File

@ -41,10 +41,7 @@ config = {
"firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
"firefox_media_branch": 'master',
"firefox_media_rev": '49b500b30b80372a6c678ec7d0a2b074844f5e84',
"firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
"firefox_ui_branch": 'mozilla-central',
"firefox_ui_rev": '32be49d74e1d10c6bf087235b1d6753c1b840bc4',
"firefox_media_rev": '0830e972e4b95fef3507207fc6bce028da27f2d3',
"suite_definitions": {
"media-tests": {

5
testing/mozharness/configs/mediatests/buildbot_windows_config.py
View File

@ -52,10 +52,7 @@ config = {
"firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
"firefox_media_branch": 'master',
"firefox_media_rev": '49b500b30b80372a6c678ec7d0a2b074844f5e84',
"firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
"firefox_ui_branch": 'mozilla-central',
"firefox_ui_rev": '32be49d74e1d10c6bf087235b1d6753c1b840bc4',
"firefox_media_rev": '0830e972e4b95fef3507207fc6bce028da27f2d3',
"suite_definitions": {
"media-tests": {

72
testing/mozharness/configs/mediatests/jenkins_config.py Executable file
View File

@ -0,0 +1,72 @@
# Default configuration as used by Mozmill CI (Jenkins)
import os
import platform
import sys
import mozharness
external_tools_path = os.path.join(
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
'external_tools',
)
config = {
'env': {
'PIP_TRUSTED_HOST': 'pypi.pub.build.mozilla.org',
},
# PIP
'find_links': ['http://pypi.pub.build.mozilla.org/pub'],
'pip_index': False,
# mozcrash support
'download_minidump_stackwalk': True,
'download_symbols': 'ondemand',
'download_tooltool': True,
# Version control information
'firefox_media_repo': 'https://github.com/mjzffr/firefox-media-tests.git',
'firefox_media_branch': 'master',
# Default test suite
'test_suite': 'media-tests',
'suite_definitions': {
'media-tests': {
'options': [],
},
'media-youtube-tests': {
'options': [
'%(test_manifest)s'
],
},
},
'default_actions': [
'clobber',
'checkout',
'download-and-extract',
'create-virtualenv',
'install',
'run-media-tests',
],
}
# General local variable overwrite
# Bug 1227079 - Python executable eeded to get it executed on Windows
if platform.system() == 'windows':
gittool = [
sys.executable,
os.path.join(external_tools_path, 'gittool.py')
]
else:
gittool = os.path.join(external_tools_path, 'gittool.py')
exes = {
'gittool.py' : gittool,
}
config['exes'] = exes

124
testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
View File

@ -4,13 +4,12 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** BEGIN LICENSE BLOCK *****
"""firefox_media_tests.py
Author: Maja Frydrychowicz
"""
import copy
import glob
import os
import re
import urlparse
from mozharness.base.log import ERROR, WARNING
from mozharness.base.script import PreScriptAction
@ -72,23 +71,14 @@ media_test_config_options = [
'dest': 'firefox_media_rev',
'help': 'which firefox_media_tests revision to use',
}],
[['--firefox-ui-repo'], {
'dest': 'firefox_ui_repo',
'default': 'https://github.com/mozilla/firefox-ui-tests.git',
'help': 'which firefox_ui_tests repo to use',
}],
[['--firefox-ui-branch'], {
'dest': 'firefox_ui_branch',
'default': 'master',
'help': 'which branch to use for firefox_ui_tests',
}],
[['--firefox-ui-rev'], {
'dest': 'firefox_ui_rev',
'help': 'which firefox_ui_tests revision to use',
}],
[["--suite"],
{"action": "store",
"dest": "test_suite",
"default": "media-tests",
"help": "suite name",
}],
] + (copy.deepcopy(testing_config_options))
class JobResultParser(TestSummaryOutputParserHelper):
""" Parses test output to determine overall result."""
def __init__(self, **kwargs):
@ -138,7 +128,9 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
actions = [
'clobber',
'checkout',
'download-and-extract',
'create-virtualenv',
'install',
'run-media-tests',
]
super(FirefoxMediaTestsBase, self).__init__(
@ -148,23 +140,45 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
**kwargs
)
c = self.config
self.media_urls = c.get('media_urls')
self.profile = c.get('profile')
self.test_timeout = int(c.get('test_timeout'))
self.tests = c.get('tests')
self.e10s = c.get('e10s')
self.installer_url = c.get('installer_url')
self.installer_path = c.get('installer_path')
self.binary_path = c.get('binary_path')
self.test_packages_url = c.get('test_packages_url')
self.test_url = c.get('test_url')
@PreScriptAction('create-virtualenv')
def _pre_create_virtualenv(self, action):
dirs = self.query_abs_dirs()
requirements_file = os.path.join(dirs['firefox_media_dir'],
marionette_requirements = os.path.join(dirs['abs_test_install_dir'],
'config',
'marionette_requirements.txt')
if os.access(marionette_requirements, os.F_OK):
self.register_virtualenv_module(requirements=[marionette_requirements],
two_pass=True)
media_tests_requirements = os.path.join(dirs['firefox_media_dir'],
'requirements.txt')
if os.path.isfile(requirements_file):
self.register_virtualenv_module(requirements=[requirements_file])
self.register_virtualenv_module(name='firefox-ui-tests',
url=dirs['firefox_ui_dir'])
self.register_virtualenv_module(name='firefox-media-tests',
url=dirs['firefox_media_dir'])
if os.access(media_tests_requirements, os.F_OK):
self.register_virtualenv_module(requirements=[media_tests_requirements],
two_pass=True)
def download_and_extract(self):
"""Overriding method from TestingMixin until firefox-media-tests are in tree.
Right now we only care about the installer and symbolds.
"""
self._download_installer()
if self.config.get('download_symbols'):
self._download_and_extract_symbols()
def query_abs_dirs(self):
if self.abs_dirs:
@ -174,8 +188,8 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
'firefox_media_dir': os.path.join(abs_dirs['abs_work_dir'],
'firefox-media-tests')
}
dirs['firefox_ui_dir'] = os.path.join(dirs['firefox_media_dir'],
'firefox-ui-tests')
dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'],
'tests')
abs_dirs.update(dirs)
self.abs_dirs = abs_dirs
return self.abs_dirs
@ -188,20 +202,13 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
self.firefox_media_vc = {
'branch': c['firefox_media_branch'],
'repo': c['firefox_media_repo'],
'revision': c['firefox_media_rev'],
'dest': dirs['firefox_media_dir'],
}
self.firefox_ui_vc = {
'branch': c['firefox_ui_branch'],
'repo': c['firefox_ui_repo'],
'revision': c['firefox_ui_rev'],
'dest': dirs['firefox_ui_dir']
}
if 'firefox-media-rev' in c:
self.firefox_media_vc['revision'] = c['firefox_media_rev']
def checkout(self):
revision = self.vcs_checkout(vcs='gittool', **self.firefox_media_vc)
if revision:
self.vcs_checkout(vcs='gittool', **self.firefox_ui_vc)
self.vcs_checkout(vcs='gittool', **self.firefox_media_vc)
def _query_cmd(self):
""" Determine how to call firefox-media-tests """
@ -226,8 +233,48 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
if self.e10s:
cmd.append('--e10s')
test_suite = self.config.get('test_suite')
if test_suite not in self.config["suite_definitions"]:
self.fatal("%s is not defined in the config!" % test_suite)
test_manifest = None if test_suite != 'media-youtube-tests' else \
os.path.join(dirs['firefox_media_dir'],
'firefox_media_tests',
'playback', 'youtube', 'manifest.ini')
config_fmt_args = {
'test_manifest': test_manifest,
}
for s in self.config["suite_definitions"][test_suite]["options"]:
cmd.append(s % config_fmt_args)
return cmd
def query_minidump_stackwalk(self):
"""We don't have an extracted test package available to get the manifest file.
So we have to explicitely download the latest version of the manifest from the
mozilla-central repository and feed it into the query_minidump_stackwalk() method.
We can remove this whole method once our tests are part of the tree.
"""
manifest_path = None
if os.environ.get('MINIDUMP_STACKWALK') or self.config.get('download_minidump_stackwalk'):
tooltool_manifest = self.query_minidump_tooltool_manifest()
url_base = 'https://hg.mozilla.org/mozilla-central/raw-file/default/testing/'
dirs = self.query_abs_dirs()
manifest_path = os.path.join(dirs['abs_work_dir'], 'releng.manifest')
try:
self.download_file(urlparse.urljoin(url_base, tooltool_manifest),
manifest_path)
except Exception as e:
self.fatal('Download of tooltool manifest file failed: %s' % e.message)
return super(FirefoxMediaTestsBase, self).query_minidump_stackwalk(manifest=manifest_path)
def run_media_tests(self):
cmd = self._query_cmd()
self.job_result_parser = JobResultParser(
@ -237,8 +284,7 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
)
env = self.query_env()
if (not os.environ.get('MINIDUMP_STACKWALK') and
self.query_minidump_stackwalk()):
if self.query_minidump_stackwalk():
env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
return_code = self.run_command(

95
testing/mozharness/scripts/firefox_media_tests_buildbot.py
View File

@ -15,8 +15,8 @@ import sys
sys.path.insert(1, os.path.dirname(sys.path[0]))
from mozharness.base.log import ERROR, DEBUG, INFO
from mozharness.base.script import PreScriptAction, PostScriptAction
from mozharness.base.log import DEBUG, ERROR, INFO
from mozharness.base.script import PostScriptAction
from mozharness.mozilla.blob_upload import (
BlobUploadMixin,
blobupload_config_options
@ -25,25 +25,16 @@ from mozharness.mozilla.buildbot import (
TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
)
from mozharness.mozilla.testing.firefox_media_tests import (
FirefoxMediaTestsBase, BUSTED, TESTFAILED, UNKNOWN, EXCEPTION, SUCCESS
FirefoxMediaTestsBase, TESTFAILED, SUCCESS
)
buildbot_media_test_options = [
[["--suite"],
{"action": "store",
"dest": "test_suite",
"default": "media-tests",
"help": "suite name",
}],
]
class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
def __init__(self):
config_options = copy.deepcopy(blobupload_config_options)
super(FirefoxMediaTestsBuildbot, self).__init__(
config_options=config_options + buildbot_media_test_options,
config_options=config_options,
all_actions=['clobber',
'read-buildbot-config',
'checkout',
@ -53,62 +44,6 @@ class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
'run-media-tests',
],
)
c = self.config
self.installer_url = c.get('installer_url')
self.installer_path = c.get('installer_path')
self.binary_path = c.get('binary_path')
self.test_packages_url = c.get('test_packages_url')
@PreScriptAction('create-virtualenv')
def _pre_create_virtualenv(self, action):
dirs = self.query_abs_dirs()
requirements = os.path.join(dirs['abs_test_install_dir'],
'config',
'marionette_requirements.txt')
if os.access(requirements, os.F_OK):
self.register_virtualenv_module(requirements=[requirements],
two_pass=True)
super(FirefoxMediaTestsBuildbot, self)._pre_create_virtualenv(action)
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
dirs['abs_blob_upload_dir'] = os.path.join(dirs['abs_work_dir'],
'blobber_upload_dir')
dirs['abs_test_install_dir'] = os.path.join(dirs['abs_work_dir'],
'tests')
self.abs_dirs = dirs
return self.abs_dirs
def _query_cmd(self):
""" Determine how to call firefox-media-tests """
cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
# configure logging
dirs = self.query_abs_dirs()
blob_upload_dir = dirs.get('abs_blob_upload_dir')
cmd += ['--gecko-log', os.path.join(blob_upload_dir,
'gecko.log')]
cmd += ['--log-html', os.path.join(blob_upload_dir,
'media_tests.html')]
cmd += ['--log-mach', os.path.join(blob_upload_dir,
'media_tests_mach.log')]
test_suite = self.config.get('test_suite')
test_manifest = None if test_suite != 'media-youtube-tests' else \
os.path.join(dirs['firefox_media_dir'],
'firefox_media_tests',
'playback', 'youtube', 'manifest.ini')
config_fmt_args = {
'test_manifest': test_manifest,
}
if test_suite not in self.config["suite_definitions"]:
self.fatal("%s is not defined in the config!" % test_suite)
for s in self.config["suite_definitions"][test_suite]["options"]:
cmd.append(s % config_fmt_args)
return cmd
def run_media_tests(self):
status = super(FirefoxMediaTestsBuildbot, self).run_media_tests()
@ -120,6 +55,28 @@ class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
tbpl_status = TBPL_WARNING
self.buildbot_status(tbpl_status)
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
abs_dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
dirs = {
'abs_blob_upload_dir': os.path.join(abs_dirs['abs_work_dir'],
'blobber_upload_dir')
}
abs_dirs.update(dirs)
self.abs_dirs = abs_dirs
return self.abs_dirs
def _query_cmd(self):
cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
dirs = self.query_abs_dirs()
# configure logging
blob_upload_dir = dirs.get('abs_blob_upload_dir')
cmd += ['--gecko-log', os.path.join(blob_upload_dir, 'gecko.log')]
cmd += ['--log-html', os.path.join(blob_upload_dir, 'media_tests.html')]
cmd += ['--log-mach', os.path.join(blob_upload_dir, 'media_tests_mach.log')]
return cmd
@PostScriptAction('run-media-tests')
def _collect_uploads(self, action, success=None):
""" Copy extra (log) files to blob upload dir. """

52
testing/mozharness/scripts/firefox_media_tests_jenkins.py Executable file
View File

@ -0,0 +1,52 @@
#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** BEGIN LICENSE BLOCK *****
"""firefox_media_tests_jenkins.py
Author: Syd Polk
"""
import copy
import glob
import os
import sys
sys.path.insert(1, os.path.dirname(sys.path[0]))
from mozharness.base.script import PreScriptAction
from mozharness.mozilla.testing.firefox_media_tests import (
FirefoxMediaTestsBase
)
class FirefoxMediaTestsJenkins(FirefoxMediaTestsBase):
def __init__(self):
super(FirefoxMediaTestsJenkins, self).__init__(
all_actions=['clobber',
'checkout',
'download-and-extract',
'create-virtualenv',
'install',
'run-media-tests',
],
)
def _query_cmd(self):
cmd = super(FirefoxMediaTestsJenkins, self)._query_cmd()
dirs = self.query_abs_dirs()
# configure logging
log_dir = dirs.get('abs_log_dir')
cmd += ['--gecko-log', os.path.join(log_dir, 'gecko.log')]
cmd += ['--log-html', os.path.join(log_dir, 'media_tests.html')]
cmd += ['--log-mach', os.path.join(log_dir, 'media_tests_mach.log')]
return cmd
if __name__ == '__main__':
media_test = FirefoxMediaTestsJenkins()
media_test.run_and_exit()