wine/gecko
0
0
Fork 0
mirror of https://gitlab.winehq.org/wine/wine-gecko.git synced 2024-09-13 09:24:08 -07:00
Code Issues Packages Projects Releases Wiki Activity

Bug 1200368 - Clean-up mozharness scripts for firefox-ui-tests. r=armenzg DONTBUILD

Browse Source
This commit is contained in:
Henrik Skupin 2015-09-03 23:11:04 +02:00
parent 3984f620e5
commit ba1e30b198
4 changed files with 615 additions and 530 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

309
testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
View File

@ -7,50 +7,114 @@
"""firefox_ui_updates.py
Author: Armen Zambrano G.
Henrik Skupin
"""
import copy
import sys
import os
import sys
import urllib2
from mozharness.base.python import (
PreScriptAction,
VirtualenvMixin,
virtualenv_config_options,
)
from mozharness.mozilla.testing.testbase import (INSTALLER_SUFFIXES)
from mozharness.mozilla.vcstools import VCSToolsScript
# General command line arguments for Firefox ui tests
firefox_ui_tests_config_options = [
[['--dry-run'], {
'dest': 'dry_run',
'default': False,
'help': 'Only show what was going to be tested.',
}],
[['--firefox-ui-branch'], {
'dest': 'firefox_ui_branch',
'help': 'which branch to use for firefox_ui_tests',
}],
[['--firefox-ui-repo'], {
'dest': 'firefox_ui_repo',
'default': 'https://github.com/mozilla/firefox-ui-tests.git',
'help': 'which firefox_ui_tests repo to use',
}],
[['--symbols-path=SYMBOLS_PATH'], {
'dest': 'symbols_path',
'help': 'absolute path to directory containing breakpad '
'symbols, or the url of a zip file containing symbols.',
}],
[['--installer-url'], {
'dest': 'installer_url',
'help': 'Point to an installer to download and test against.',
}],
[['--installer-path'], {
'dest': 'installer_path',
'help': 'Point to an installer to test against.',
}],
] + copy.deepcopy(virtualenv_config_options)
# Command line arguments for update tests
firefox_ui_update_harness_config_options = [
[['--update-allow-mar-channel'], {
'dest': 'update_allow_mar_channel',
'help': 'Additional MAR channel to be allowed for updates, e.g. '
'"firefox-mozilla-beta" for updating a release build to '
'the latest beta build.',
}],
[['--update-channel'], {
'dest': 'update_channel',
'help': 'Update channel to use.',
}],
[['--update-direct-only'], {
'action': 'store_true',
'dest': 'update_direct_only',
'help': 'Only perform a direct update.',
}],
[['--update-fallback-only'], {
'action': 'store_true',
'dest': 'update_fallback_only',
'help': 'Only perform a fallback update.',
}],
[['--update-override-url'], {
'dest': 'update_override_url',
'help': 'Force specified URL to use for update checks.',
}],
[['--update-target-buildid'], {
'dest': 'update_target_buildid',
'help': 'Build ID of the updated build',
}],
[['--update-target-version'], {
'dest': 'update_target_version',
'help': 'Version of the updated build.',
}],
]
firefox_ui_update_config_options = firefox_ui_update_harness_config_options \
+ copy.deepcopy(firefox_ui_tests_config_options)
class FirefoxUITests(VCSToolsScript, VirtualenvMixin):
config_options = [
[['--firefox-ui-repo'], {
'dest': 'firefox_ui_repo',
'default': 'https://github.com/mozilla/firefox-ui-tests.git',
'help': 'which firefox_ui_tests repo to use',
}],
[['--firefox-ui-branch'], {
'dest': 'firefox_ui_branch',
'help': 'which branch to use for firefox_ui_tests',
}],
] + copy.deepcopy(virtualenv_config_options)
cli_script = 'firefox-ui-tests'
def __init__(self, config_options=[], all_actions=[], **kwargs):
self.config_options += config_options
def __init__(self, config_options=None,
all_actions=None, default_actions=None,
*args, **kwargs):
config_options = config_options or firefox_ui_tests_config_options
actions = [
'clobber',
'checkout',
'create-virtualenv',
'run-tests',
]
if all_actions is None:
# Default actions
all_actions = [
'clobber',
'checkout',
'create-virtualenv',
'run-tests',
]
super(FirefoxUITests, self).__init__(
config_options=self.config_options,
all_actions=all_actions,
**kwargs
)
VCSToolsScript.__init__(self,
config_options=config_options,
all_actions=all_actions or actions,
default_actions=default_actions or actions,
*args, **kwargs)
VirtualenvMixin.__init__(self)
self.firefox_ui_repo = self.config['firefox_ui_repo']
self.firefox_ui_branch = self.config.get('firefox_ui_branch')
@ -60,34 +124,15 @@ class FirefoxUITests(VCSToolsScript, VirtualenvMixin):
'Please specify --firefox-ui-branch. Valid values can be found '
'in here https://github.com/mozilla/firefox-ui-tests/branches')
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
abs_dirs = super(FirefoxUITests, self).query_abs_dirs()
self.installer_url = self.config.get('installer_url')
self.installer_path = self.config.get('installer_path')
dirs = {
'fx_ui_dir': os.path.join(abs_dirs['abs_work_dir'], 'firefox_ui_tests'),
}
abs_dirs.update(dirs)
self.abs_dirs = abs_dirs
return self.abs_dirs
def checkout(self):
'''
We checkout firefox_ui_tests and update to the right branch
for it.
'''
dirs = self.query_abs_dirs()
self.vcs_checkout(
repo=self.firefox_ui_repo,
dest=dirs['fx_ui_dir'],
branch=self.firefox_ui_branch,
vcs='gittool'
)
if self.installer_path:
self.installer_path = os.path.abspath(self.installer_path)
if not os.path.exists(self.installer_path):
self.critical('Please make sure that the path to the installer exists.')
sys.exit(1)
@PreScriptAction('create-virtualenv')
def _pre_create_virtualenv(self, action):
@ -98,6 +143,160 @@ class FirefoxUITests(VCSToolsScript, VirtualenvMixin):
url=dirs['fx_ui_dir'],
)
def _query_symbols_url(self, installer_url):
for suffix in INSTALLER_SUFFIXES:
if installer_url.endswith(suffix):
symbols_url = installer_url[:-len(suffix)] + '.crashreporter-symbols.zip'
continue
if symbols_url:
self.info('Symbols_url: {}'.format(symbols_url))
if not symbols_url.startswith('http'):
return symbols_url
try:
# Let's see if the symbols are available
urllib2.urlopen(symbols_url)
return symbols_url
except urllib2.HTTPError, e:
self.warning('{} - {}'.format(str(e), symbols_url))
return None
else:
self.fatal('Can\'t find symbols_url from installer_url: {}!'.format(installer_url))
def checkout(self):
"""
We checkout firefox_ui_tests and update to the right branch
for it.
"""
dirs = self.query_abs_dirs()
self.vcs_checkout(
repo=self.firefox_ui_repo,
dest=dirs['fx_ui_dir'],
branch=self.firefox_ui_branch,
vcs='gittool'
)
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
abs_dirs = VCSToolsScript.query_abs_dirs(self)
abs_dirs.update({
'fx_ui_dir': os.path.join(abs_dirs['abs_work_dir'], 'firefox_ui_tests'),
})
self.abs_dirs = abs_dirs
return self.abs_dirs
def query_extra_cmd_args(self):
"""Collects specific update test related command line arguments.
Sub classes should override this method for their own specific arguments.
"""
return []
def run_test(self, installer_path, script_name, env=None, symbols_url=None,
cleanup=True, marionette_port=2828):
"""All required steps for running the tests against an installer."""
dirs = self.query_abs_dirs()
bin_dir = os.path.dirname(self.query_python_path())
fx_ui_tests_bin = os.path.join(bin_dir, script_name)
gecko_log = os.path.join(dirs['abs_log_dir'], 'gecko.log')
# Build the command
cmd = [
fx_ui_tests_bin,
'--installer', installer_path,
# Log to stdout until tests are stable.
'--gecko-log=-',
'--address', 'localhost:{}'.format(marionette_port),
# Use the work dir to get temporary data stored
'--workspace', dirs['abs_work_dir'],
]
if symbols_url:
cmd += ['--symbols-path', symbols_url]
# Collect all pass-through harness options to the script
cmd.extend(self.query_extra_cmd_args())
return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'],
output_timeout=300, env=env)
# Return more output if we fail
if return_code:
if os.path.exists(gecko_log):
contents = self.read_from_file(gecko_log, verbose=False)
self.warning('== Dumping gecko output ==')
self.warning(contents)
self.warning('== End of gecko output ==')
else:
# We're outputting to stdout with --gecko-log=- so there is not log to
# complaing about. Remove the commented line below when changing
# this behaviour.
# self.warning('No gecko.log was found: %s' % gecko_log)
pass
if cleanup:
for filepath in (installer_path, gecko_log):
if os.path.exists(filepath):
self.debug('Removing {}'.format(filepath))
os.remove(filepath)
return return_code
@PreScriptAction('run-tests')
def _pre_run_tests(self, action):
if not self.installer_path and not self.installer_url:
self.critical('Please specify an installer via --installer-path or --installer-url.')
sys.exit(1)
def run_tests(self):
pass
dirs = self.query_abs_dirs()
if self.installer_url:
self.installer_path = self.download_file(
self.installer_url,
parent_dir=dirs['abs_work_dir']
)
symbols_url = self._query_symbols_url(installer_url=self.installer_path)
return self.run_test(
installer_path=self.installer_path,
script_name=self.cli_script,
env=self.query_env(),
symbols_url=symbols_url,
cleanup=False,
)
class FirefoxUIUpdateTests(FirefoxUITests):
cli_script = 'firefox-ui-update'
def __init__(self, config_options=None, *args, **kwargs):
config_options = config_options or firefox_ui_update_config_options
FirefoxUITests.__init__(self, config_options=config_options,
*args, **kwargs)
def query_extra_cmd_args(self):
"""Collects specific update test related command line arguments."""
args = []
for option in firefox_ui_update_harness_config_options:
dest = option[1]['dest']
name = self.config.get(dest)
if name:
if type(name) is bool:
args.append(option[0][0])
else:
args.extend([option[0][0], self.config[dest]])
return args

23
testing/mozharness/scripts/firefox_ui_tests/update.py Executable file
View File

@ -0,0 +1,23 @@
#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""firefox_ui_updates.py
Author: Armen Zambrano G.
Henrik Skupin
"""
import os
import sys
# load modules from parent dir
sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUIUpdateTests
if __name__ == '__main__':
myScript = FirefoxUIUpdateTests()
myScript.run_and_exit()

338
testing/mozharness/scripts/firefox_ui_tests/update_release.py Executable file
View File

@ -0,0 +1,338 @@
#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""firefox_ui_updates.py
Author: Armen Zambrano G.
Henrik Skupin
"""
import copy
import os
import pprint
import re
import sys
import urllib
# load modules from parent dir
sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
from mozharness.base.python import PreScriptAction
from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, EXIT_STATUS_DICT
from mozharness.mozilla.testing.firefox_ui_tests import (
FirefoxUIUpdateTests,
firefox_ui_update_config_options
)
# Command line arguments for release update tests
firefox_ui_update_release_config_options = [
[['--build-number'], {
'dest': 'build_number',
'help': 'Build number of release, eg: 2',
}],
[['--limit-locales'], {
'dest': 'limit_locales',
'default': -1,
'type': int,
'help': 'Limit the number of locales to run.',
}],
[['--release-update-config'], {
'dest': 'release_update_config',
'help': 'Name of the release update verification config file to use.',
}],
[['--this-chunk'], {
'dest': 'this_chunk',
'default': 1,
'help': 'What chunk of locales to process.',
}],
[['--tools-repo'], {
'dest': 'tools_repo',
'default': 'http://hg.mozilla.org/build/tools',
'help': 'Which tools repo to check out',
}],
[['--tools-tag'], {
'dest': 'tools_tag',
'help': 'Which revision/tag to use for the tools repository.',
}],
[['--total-chunks'], {
'dest': 'total_chunks',
'default': 1,
'help': 'Total chunks to dive the locales into.',
}],
] + copy.deepcopy(firefox_ui_update_config_options)
class ReleaseFirefoxUIUpdateTests(FirefoxUIUpdateTests):
def __init__(self):
all_actions = [
'clobber',
'checkout',
'create-virtualenv',
'read-release-update-config',
'run-tests',
]
FirefoxUIUpdateTests.__init__(self, all_actions=all_actions,
default_actions=all_actions,
config_options=firefox_ui_update_release_config_options,
append_env_variables_from_configs=True)
self.tools_repo = self.config.get('tools_repo')
self.tools_tag = self.config.get('tools_tag')
assert self.tools_repo and self.tools_tag, \
'Without the "--tools-tag" we can\'t clone the releng\'s tools repository.'
self.limit_locales = int(self.config.get('limit_locales'))
# This will be a list containing one item per release based on configs
# from tools/release/updates/*cfg
self.releases = None
def _modify_url(self, rel_info):
# This is a temporary hack to find crash symbols. It should be replaced
# with something that doesn't make wild guesses about where symbol
# packages are.
# We want this:
# https://ftp.mozilla.org/pub/mozilla.org/firefox/candidates/40.0b1-candidates/build1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip
# https://ftp.mozilla.org/pub/mozilla.org//firefox/releases/40.0b1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip
installer_from = rel_info['from']
version = (re.search('/firefox/releases/(%s.*)\/.*\/.*\/.*' % rel_info['release'],
installer_from)).group(1)
temp_from = installer_from.replace(version, '%s-candidates/build%s' % (
version, self.config['build_number']),
1).replace('releases', 'candidates')
temp_url = rel_info['ftp_server_from'] + urllib.quote(temp_from.replace('%locale%',
'en-US'))
self.info('Installer url under stage/candidates dir: {}'.format(temp_url))
return temp_url
def checkout(self):
"""
We checkout the tools repository and update to the right branch
for it.
"""
dirs = self.query_abs_dirs()
FirefoxUIUpdateTests.checkout(self)
self.vcs_checkout(
repo=self.tools_repo,
dest=dirs['abs_tools_dir'],
revision=self.tools_tag,
vcs='hgtool'
)
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
abs_dirs = FirefoxUIUpdateTests.query_abs_dirs(self)
abs_dirs.update({
'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'),
})
self.abs_dirs = abs_dirs
return self.abs_dirs
def read_release_update_config(self):
'''
Builds a testing matrix based on an update verification configuration
file under the tools repository (release/updates/*.cfg).
Each release info line of the update verification files look similar to the following.
NOTE: This shows each pair of information as a new line but in reality
there is one white space separting them. We only show the values we care for.
release="38.0"
platform="Linux_x86_64-gcc3"
build_id="20150429135941"
locales="ach af ... zh-TW"
channel="beta-localtest"
from="/firefox/releases/38.0b9/linux-x86_64/%locale%/firefox-38.0b9.tar.bz2"
ftp_server_from="http://stage.mozilla.org/pub/mozilla.org"
We will store this information in self.releases as a list of releases.
NOTE: We will talk of full and quick releases. Full release info normally contains a subset
of all locales (except for the most recent releases). A quick release has all locales,
however, it misses the fields 'from' and 'ftp_server_from'.
Both pairs of information complement each other but differ in such manner.
'''
dirs = self.query_abs_dirs()
assert os.path.exists(dirs['abs_tools_dir']), \
'Without the tools/ checkout we can\'t use releng\'s config parser.'
if self.config.get('release_update_config'):
# The config file is part of the tools repository. Make sure that if specified
# we force a revision of that repository to be set.
if self.tools_tag is None:
self.fatal('Make sure to specify the --tools-tag')
self.release_update_config = self.config['release_update_config']
# Import the config parser
sys.path.insert(1, os.path.join(dirs['abs_tools_dir'], 'lib', 'python'))
from release.updates.verify import UpdateVerifyConfig
uvc = UpdateVerifyConfig()
config_file = os.path.join(dirs['abs_tools_dir'], 'release', 'updates',
self.config['release_update_config'])
uvc.read(config_file)
if not hasattr(self, 'update_channel'):
self.update_channel = uvc.channel
# Filter out any releases that are less than Gecko 38
uvc.releases = [r for r in uvc.releases
if int(r['release'].split('.')[0]) >= 38]
temp_releases = []
for rel_info in uvc.releases:
# This is the full release info
if 'from' in rel_info and rel_info['from'] is not None:
# Let's find the associated quick release which contains the remaining locales
# for all releases except for the most recent release which contain all locales
quick_release = uvc.getRelease(build_id=rel_info['build_id'], from_path=None)
if quick_release != {}:
rel_info['locales'] = sorted(rel_info['locales'] + quick_release['locales'])
temp_releases.append(rel_info)
uvc.releases = temp_releases
chunked_config = uvc.getChunk(
chunks=int(self.config['total_chunks']),
thisChunk=int(self.config['this_chunk'])
)
self.releases = chunked_config.releases
@PreScriptAction('run-tests')
def _pre_run_tests(self, action):
assert 'release_update_config' in self.config, \
'You have to specify --release-update-config.'
def run_tests(self):
dirs = self.query_abs_dirs()
# We don't want multiple outputs of the same environment information. To prevent
# that, we can't make it an argument of run_command and have to print it on our own.
self.info('Using env: {}'.format(pprint.pformat(self.query_env())))
results = {}
locales_counter = 0
for rel_info in sorted(self.releases, key=lambda release: release['build_id']):
build_id = rel_info['build_id']
results[build_id] = {}
self.info('About to run {buildid} {path} - {num_locales} locales'.format(
buildid=build_id,
path=rel_info['from'],
num_locales=len(rel_info['locales'])
))
# Each locale gets a fresh port to avoid address in use errors in case of
# tests that time out unexpectedly.
marionette_port = 2827
for locale in rel_info['locales']:
locales_counter += 1
self.info('Running {buildid} {locale}'.format(buildid=build_id,
locale=locale))
if self.limit_locales > -1 and locales_counter > self.limit_locales:
self.info('We have reached the limit of locales we were intending to run')
break
if self.config['dry_run']:
continue
# Safe temporary hack to determine symbols URL from en-US
# build1 in the candidates dir
ftp_candidates_installer_url = self._modify_url(rel_info)
symbols_url = self._query_symbols_url(
installer_url=ftp_candidates_installer_url)
# Determine from where to download the file
installer_url = '{server}/{fragment}'.format(
server=rel_info['ftp_server_from'],
fragment=urllib.quote(rel_info['from'].replace('%locale%', locale))
)
installer_path = self.download_file(
url=installer_url,
parent_dir=dirs['abs_work_dir']
)
marionette_port += 1
retcode = self.run_test(
installer_path=installer_path,
script_name=self.cli_script,
env=self.query_env(avoid_host_env=True),
symbols_url=symbols_url,
marionette_port=marionette_port,
)
if retcode:
self.warning('FAIL: {} has failed.'.format(sys.argv[0]))
base_cmd = 'python {command} --firefox-ui-branch {branch} ' \
'--release-update-config {config} --tools-tag {tag}'.format(
command=sys.argv[0],
branch=self.firefox_ui_branch,
config=self.release_update_config,
tag=self.tools_tag
)
for config in self.config['config_files']:
base_cmd += ' --cfg {}'.format(config)
if symbols_url:
base_cmd += ' --symbols-path {}'.format(symbols_url)
base_cmd += ' --installer-url {}'.format(installer_url)
self.info('You can run the *specific* locale on the same machine with:')
self.info(base_cmd)
self.info('You can run the *specific* locale on *your* machine with:')
self.info('{} --cfg developer_config.py'.format(base_cmd))
results[build_id][locale] = retcode
self.info('Completed {buildid} {locale} with return code: {retcode}'.format(
buildid=build_id,
locale=locale,
retcode=retcode))
if self.limit_locales > -1 and locales_counter > self.limit_locales:
break
# Determine which locales have failed and set scripts exit code
exit_status = TBPL_SUCCESS
for build_id in sorted(results.keys()):
failed_locales = []
for locale in sorted(results[build_id].keys()):
if results[build_id][locale] != 0:
failed_locales.append(locale)
if failed_locales:
if exit_status == TBPL_SUCCESS:
self.info('\nSUMMARY - Failed locales for {}:'.format(self.cli_script))
self.info('====================================================')
exit_status = TBPL_WARNING
self.info(build_id)
self.info(' {}'.format(', '.join(failed_locales)))
self.return_code = EXIT_STATUS_DICT[exit_status]
if __name__ == '__main__':
myScript = ReleaseFirefoxUIUpdateTests()
myScript.run_and_exit()

475
testing/mozharness/scripts/firefox_ui_updates.py
View File

@ -1,475 +0,0 @@
#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""firefox_ui_updates.py
Author: Armen Zambrano G.
"""
import copy
import os
import re
import urllib
import urllib2
import sys
# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))
from mozharness.base.script import PreScriptAction
from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUITests
from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, EXIT_STATUS_DICT
INSTALLER_SUFFIXES = ('.tar.bz2', '.zip', '.dmg', '.exe', '.apk', '.tar.gz')
class FirefoxUIUpdates(FirefoxUITests):
# This will be a list containing one item per release based on configs
# from tools/release/updates/*cfg
releases = None
channel = None
harness_extra_args = [
[['--update-allow-mar-channel'], {
'dest': 'update_allow_mar_channel',
'help': 'Additional MAR channel to be allowed for updates, e.g. '
'"firefox-mozilla-beta" for updating a release build to '
'the latest beta build.',
}],
[['--update-channel'], {
'dest': 'update_channel',
'help': 'Update channel to use.',
}],
[['--update-target-version'], {
'dest': 'update_target_version',
'help': 'Version of the updated build.',
}],
[['--update-target-buildid'], {
'dest': 'update_target_buildid',
'help': 'Build ID of the updated build',
}],
[['--symbols-path=SYMBOLS_PATH'], {
'dest': 'symbols_path',
'help': 'absolute path to directory containing breakpad '
'symbols, or the url of a zip file containing symbols.',
}],
]
def __init__(self):
config_options = [
[['--tools-repo'], {
'dest': 'tools_repo',
'default': 'http://hg.mozilla.org/build/tools',
'help': 'Which tools repo to check out',
}],
[['--tools-tag'], {
'dest': 'tools_tag',
'help': 'Which revision/tag to use for the tools repository.',
}],
[['--update-verify-config'], {
'dest': 'update_verify_config',
'help': 'Which update verify config file to use.',
}],
[['--this-chunk'], {
'dest': 'this_chunk',
'default': 1,
'help': 'What chunk of locales to process.',
}],
[['--total-chunks'], {
'dest': 'total_chunks',
'default': 1,
'help': 'Total chunks to dive the locales into.',
}],
[['--dry-run'], {
'dest': 'dry_run',
'default': False,
'help': 'Only show what was going to be tested.',
}],
[["--build-number"], {
"dest": "build_number",
"help": "Build number of release, eg: 2",
}],
# These are options when we don't use the releng update config file
[['--installer-url'], {
'dest': 'installer_url',
'help': 'Point to an installer to download and test against.',
}],
[['--installer-path'], {
'dest': 'installer_path',
'help': 'Point to an installer to test against.',
}],
[['--limit-locales'], {
'dest': 'limit_locales',
'default': -1,
'type': int,
'help': 'Limit the number of locales to run.',
}],
] + copy.deepcopy(self.harness_extra_args)
super(FirefoxUIUpdates, self).__init__(
config_options=config_options,
all_actions=[
'clobber',
'checkout',
'create-virtualenv',
'determine-testing-configuration',
'run-tests',
],
append_env_variables_from_configs=True,
)
dirs = self.query_abs_dirs()
self.limit_locales = int(self.config.get('limit_locales'))
self.tools_repo = self.config['tools_repo']
self.tools_tag = self.config.get('tools_tag')
if self.config.get('update_verify_config'):
self.update_verify_config = self.config['update_verify_config']
self.updates_config_file = os.path.join(
dirs['abs_tools_dir'], 'release', 'updates',
self.config['update_verify_config']
)
# We want to make sure that anyone trying to reproduce a job will
# is using the exact tools tag for reproducibility's sake
if self.config.get('tools_tag') is None:
self.fatal('Make sure to specify the --tools-tag')
self.installer_url = self.config.get('installer_url')
self.installer_path = self.config.get('installer_path')
if self.installer_path:
self.installer_path = os.path.abspath(self.installer_path)
if not os.path.exists(self.installer_path):
self.critical('Please make sure that the path to the installer exists.')
exit(1)
assert ('update_verify_config' in self.config or
self.installer_url or self.installer_path,
'Either specify --update-verify-config, --installer-url or --installer-path.')
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
abs_dirs = super(FirefoxUIUpdates, self).query_abs_dirs()
dirs = {
'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'),
}
abs_dirs.update(dirs)
self.abs_dirs = abs_dirs
return self.abs_dirs
def checkout(self):
'''
This checkouts firefox_ui_tests and update to the right branch
for it.
We also checkout the tools repo if requested because it contains the
configuration files about which locales to test.
'''
super(FirefoxUIUpdates, self).checkout()
dirs = self.query_abs_dirs()
if self.tools_tag:
self.vcs_checkout(
repo=self.tools_repo,
dest=dirs['abs_tools_dir'],
revision=self.tools_tag,
vcs='hgtool'
)
def determine_testing_configuration(self):
'''
This method builds a testing matrix either based on an update verification
configuration file under the tools repo (release/updates/*.cfg)
OR it skips it when we use --installer-url --installer-path
Each release info line of the update verification files look similar to the following.
NOTE: This shows each pair of information as a new line but in reality
there is one white space separting them. We only show the values we care for.
release="38.0"
platform="Linux_x86_64-gcc3"
build_id="20150429135941"
locales="ach af ... zh-TW"
channel="beta-localtest"
from="/firefox/releases/38.0b9/linux-x86_64/%locale%/firefox-38.0b9.tar.bz2"
ftp_server_from="http://stage.mozilla.org/pub/mozilla.org"
We will store this information in self.releases as a list of releases.
NOTE: We will talk of full and quick releases. Full release info normally contains a subset
of all locales (except for the most recent releases). A quick release has all locales,
however, it misses the fields 'from' and 'ftp_server_from'.
Both pairs of information complement each other but differ in such manner.
'''
if self.installer_url or self.installer_path:
return
dirs = self.query_abs_dirs()
assert os.path.exists(dirs['abs_tools_dir']), \
"Without the tools/ checkout we can't use releng's config parser."
# Import the config parser
sys.path.insert(1, os.path.join(dirs['abs_tools_dir'], 'lib', 'python'))
from release.updates.verify import UpdateVerifyConfig
uvc = UpdateVerifyConfig()
uvc.read(self.updates_config_file)
self.channel = uvc.channel
# Filter out any releases that are less than Gecko 38
uvc.releases = [r for r in uvc.releases
if int(r["release"].split('.')[0]) >= 38]
temp_releases = []
for rel_info in uvc.releases:
# This is the full release info
if 'from' in rel_info and rel_info['from'] is not None:
# Let's find the associated quick release which contains the remaining locales
# for all releases except for the most recent release which contain all locales
quick_release = uvc.getRelease(build_id=rel_info['build_id'], from_path=None)
if quick_release != {}:
rel_info['locales'] = sorted(rel_info['locales'] + quick_release['locales'])
temp_releases.append(rel_info)
uvc.releases = temp_releases
chunked_config = uvc.getChunk(
chunks=int(self.config['total_chunks']),
thisChunk=int(self.config['this_chunk'])
)
self.releases = chunked_config.releases
def _modify_url(self, rel_info):
# This is a temporary hack to find crash symbols. It should be replaced
# with something that doesn't make wild guesses about where symbol
# packages are.
# We want this:
# https://ftp.mozilla.org/pub/mozilla.org/firefox/candidates/40.0b1-candidates/build1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip
# https://ftp.mozilla.org/pub/mozilla.org//firefox/releases/40.0b1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip
installer_from = rel_info['from']
version = (re.search('/firefox/releases/(%s.*)\/.*\/.*\/.*' % rel_info['release'], installer_from)).group(1)
temp_from = installer_from.replace(version, '%s-candidates/build%s' % (version, self.config["build_number"]), 1).replace('releases', 'candidates')
temp_url = rel_info["ftp_server_from"] + urllib.quote(temp_from.replace('%locale%', 'en-US'))
self.info('Installer url under stage/candidates dir %s' % temp_url)
return temp_url
def _query_symbols_url(self, installer_url):
for suffix in INSTALLER_SUFFIXES:
if installer_url.endswith(suffix):
symbols_url = installer_url[:-len(suffix)] + '.crashreporter-symbols.zip'
continue
if symbols_url:
self.info('Candidate symbols_url: %s' % symbols_url)
if not symbols_url.startswith('http'):
return symbols_url
try:
# Let's see if the symbols are available
urllib2.urlopen(symbols_url)
return symbols_url
except urllib2.HTTPError, e:
self.warning("%s - %s" % (str(e), symbols_url))
return None
else:
self.fatal("Can't figure out symbols_url from installer_url %s!" % installer_url)
@PreScriptAction('run-tests')
def _pre_run_tests(self, action):
if self.releases is None and not (self.installer_url or self.installer_path):
self.fatal('You need to call --determine-testing-configuration as well.')
def _run_test(self, installer_path, symbols_url=None, update_channel=None, cleanup=True,
marionette_port=2828):
'''
All required steps for running the tests against an installer.
'''
dirs = self.query_abs_dirs()
env = self.query_env(avoid_host_env=True)
bin_dir = os.path.dirname(self.query_python_path())
fx_ui_tests_bin = os.path.join(bin_dir, 'firefox-ui-update')
gecko_log = os.path.join(dirs['abs_work_dir'], 'gecko.log')
# Build the command
cmd = [
fx_ui_tests_bin,
'--installer', installer_path,
# Log to stdout until tests are stable.
'--gecko-log=-',
'--address=localhost:%s' % marionette_port,
# Use the work dir to get temporary data stored
'--workspace=%s' % dirs['abs_work_dir'],
]
if symbols_url:
cmd += ['--symbols-path', symbols_url]
for arg in self.harness_extra_args:
dest = arg[1]['dest']
if dest in self.config:
cmd += [' '.join(arg[0]), self.config[dest]]
if update_channel:
cmd += ['--update-channel', update_channel]
return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'],
output_timeout=300,
env=env)
# Return more output if we fail
if return_code != 0:
self.info('Internally this is the command fx-ui-updates executed')
self.info('%s' % ' '.join(map(str, cmd)))
if os.path.exists(gecko_log):
contents = self.read_from_file(gecko_log, verbose=False)
self.warning('== Dumping gecko output ==')
self.warning(contents)
self.warning('== End of gecko output ==')
else:
# We're outputting to stdout with --gecko-log=- so there is not log to
# complaing about. Remove the commented line below when changing
# this behaviour.
# self.warning('No gecko.log was found: %s' % gecko_log)
pass
if cleanup:
for filepath in (installer_path, gecko_log):
if os.path.exists(filepath):
self.debug('Removing %s' % filepath)
os.remove(filepath)
return return_code
def run_tests(self):
dirs = self.query_abs_dirs()
if self.installer_url or self.installer_path:
if self.installer_url:
self.installer_path = self.download_file(
self.installer_url,
parent_dir=dirs['abs_work_dir']
)
symbols_url = self._query_symbols_url(installer_url=self.installer_path)
return self._run_test(
installer_path=self.installer_path,
symbols_url=symbols_url,
cleanup=False
)
else:
results = {}
locales_counter = 0
for rel_info in sorted(self.releases, key=lambda release: release['build_id']):
build_id = rel_info['build_id']
results[build_id] = {}
self.info('About to run %s %s - %s locales' % (
build_id,
rel_info['from'],
len(rel_info['locales'])
))
# Each locale gets a fresh port to avoid address in use errors in case of
# tests that time out unexpectedly.
marionette_port = 2827
for locale in rel_info['locales']:
locales_counter += 1
self.info("Running %s %s" % (build_id, locale))
if self.limit_locales and locales_counter > self.limit_locales:
self.info("We have reached the limit of locales we were intending to run")
break
if self.config['dry_run']:
continue
# Safe temp hack to determine symbols URL from en-US build1 in the candidates dir
ftp_candidates_installer_url = self._modify_url(rel_info)
symbols_url = self._query_symbols_url(installer_url=ftp_candidates_installer_url)
# Determine from where to download the file
installer_url = '%s/%s' % (
rel_info['ftp_server_from'],
urllib.quote(rel_info['from'].replace('%locale%', locale))
)
installer_path = self.download_file(
url=installer_url,
parent_dir=dirs['abs_work_dir']
)
marionette_port += 1
retcode = self._run_test(
installer_path=installer_path,
symbols_url=symbols_url,
update_channel=self.channel,
marionette_port=marionette_port)
if retcode != 0:
self.warning('FAIL: firefox-ui-update has failed.')
base_cmd = 'python scripts/firefox_ui_updates.py'
for c in self.config['config_files']:
base_cmd += ' --cfg %s' % c
base_cmd += ' --firefox-ui-branch %s --update-verify-config %s --tools-tag %s' % \
(self.firefox_ui_branch, self.update_verify_config, self.tools_tag)
base_cmd += ' --installer-url %s' % installer_url
if symbols_url:
base_cmd += ' --symbols-path %s' % symbols_url
self.info('You can run the *specific* locale on the same machine with:')
self.info('%s' % base_cmd)
self.info('You can run the *specific* locale on *your* machine with:')
self.info('%s --cfg developer_config.py' % base_cmd)
results[build_id][locale] = retcode
self.info("Completed %s %s with return code: %s" % (build_id, locale, retcode))
if self.limit_locales and locales_counter > self.limit_locales:
break
# Determine which locales have failed and set scripts exit code
exit_status = TBPL_SUCCESS
for build_id in sorted(results.keys()):
failed_locales = []
for locale in sorted(results[build_id].keys()):
if results[build_id][locale] != 0:
failed_locales.append(locale)
if failed_locales:
if exit_status == TBPL_SUCCESS:
self.info("")
self.info("SUMMARY - Firefox UI update tests failed locales:")
self.info("=================================================")
exit_status = TBPL_WARNING
self.info(build_id)
self.info(" %s" % (', '.join(failed_locales)))
self.return_code = EXIT_STATUS_DICT[exit_status]
if __name__ == '__main__':
myScript = FirefoxUIUpdates()
myScript.run_and_exit()