From ba1e30b198ce0444702fcffeaf6144879060a749 Mon Sep 17 00:00:00 2001 From: Henrik Skupin Date: Thu, 3 Sep 2015 23:11:04 +0200 Subject: [PATCH] Bug 1200368 - Clean-up mozharness scripts for firefox-ui-tests. r=armenzg DONTBUILD --- .../mozilla/testing/firefox_ui_tests.py | 309 ++++++++++-- .../scripts/firefox_ui_tests/update.py | 23 + .../firefox_ui_tests/update_release.py | 338 +++++++++++++ .../mozharness/scripts/firefox_ui_updates.py | 475 ------------------ 4 files changed, 615 insertions(+), 530 deletions(-) create mode 100755 testing/mozharness/scripts/firefox_ui_tests/update.py create mode 100755 testing/mozharness/scripts/firefox_ui_tests/update_release.py delete mode 100755 testing/mozharness/scripts/firefox_ui_updates.py diff --git a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py index bfdc6ef387d..7d9248e757b 100644 --- a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py +++ b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py @@ -7,50 +7,114 @@ """firefox_ui_updates.py Author: Armen Zambrano G. + Henrik Skupin """ import copy -import sys import os +import sys +import urllib2 from mozharness.base.python import ( PreScriptAction, VirtualenvMixin, virtualenv_config_options, ) +from mozharness.mozilla.testing.testbase import (INSTALLER_SUFFIXES) from mozharness.mozilla.vcstools import VCSToolsScript +# General command line arguments for Firefox ui tests +firefox_ui_tests_config_options = [ + [['--dry-run'], { + 'dest': 'dry_run', + 'default': False, + 'help': 'Only show what was going to be tested.', + }], + [['--firefox-ui-branch'], { + 'dest': 'firefox_ui_branch', + 'help': 'which branch to use for firefox_ui_tests', + }], + [['--firefox-ui-repo'], { + 'dest': 'firefox_ui_repo', + 'default': 'https://github.com/mozilla/firefox-ui-tests.git', + 'help': 'which firefox_ui_tests repo to use', + }], + [['--symbols-path=SYMBOLS_PATH'], { + 'dest': 'symbols_path', + 'help': 'absolute path to directory containing breakpad ' + 'symbols, or the url of a zip file containing symbols.', + }], + [['--installer-url'], { + 'dest': 'installer_url', + 'help': 'Point to an installer to download and test against.', + }], + [['--installer-path'], { + 'dest': 'installer_path', + 'help': 'Point to an installer to test against.', + }], +] + copy.deepcopy(virtualenv_config_options) + +# Command line arguments for update tests +firefox_ui_update_harness_config_options = [ + [['--update-allow-mar-channel'], { + 'dest': 'update_allow_mar_channel', + 'help': 'Additional MAR channel to be allowed for updates, e.g. ' + '"firefox-mozilla-beta" for updating a release build to ' + 'the latest beta build.', + }], + [['--update-channel'], { + 'dest': 'update_channel', + 'help': 'Update channel to use.', + }], + [['--update-direct-only'], { + 'action': 'store_true', + 'dest': 'update_direct_only', + 'help': 'Only perform a direct update.', + }], + [['--update-fallback-only'], { + 'action': 'store_true', + 'dest': 'update_fallback_only', + 'help': 'Only perform a fallback update.', + }], + [['--update-override-url'], { + 'dest': 'update_override_url', + 'help': 'Force specified URL to use for update checks.', + }], + [['--update-target-buildid'], { + 'dest': 'update_target_buildid', + 'help': 'Build ID of the updated build', + }], + [['--update-target-version'], { + 'dest': 'update_target_version', + 'help': 'Version of the updated build.', + }], +] + +firefox_ui_update_config_options = firefox_ui_update_harness_config_options \ + + copy.deepcopy(firefox_ui_tests_config_options) + + class FirefoxUITests(VCSToolsScript, VirtualenvMixin): - config_options = [ - [['--firefox-ui-repo'], { - 'dest': 'firefox_ui_repo', - 'default': 'https://github.com/mozilla/firefox-ui-tests.git', - 'help': 'which firefox_ui_tests repo to use', - }], - [['--firefox-ui-branch'], { - 'dest': 'firefox_ui_branch', - 'help': 'which branch to use for firefox_ui_tests', - }], - ] + copy.deepcopy(virtualenv_config_options) + cli_script = 'firefox-ui-tests' - def __init__(self, config_options=[], all_actions=[], **kwargs): - self.config_options += config_options + def __init__(self, config_options=None, + all_actions=None, default_actions=None, + *args, **kwargs): + config_options = config_options or firefox_ui_tests_config_options + actions = [ + 'clobber', + 'checkout', + 'create-virtualenv', + 'run-tests', + ] - if all_actions is None: - # Default actions - all_actions = [ - 'clobber', - 'checkout', - 'create-virtualenv', - 'run-tests', - ] - - super(FirefoxUITests, self).__init__( - config_options=self.config_options, - all_actions=all_actions, - **kwargs - ) + VCSToolsScript.__init__(self, + config_options=config_options, + all_actions=all_actions or actions, + default_actions=default_actions or actions, + *args, **kwargs) + VirtualenvMixin.__init__(self) self.firefox_ui_repo = self.config['firefox_ui_repo'] self.firefox_ui_branch = self.config.get('firefox_ui_branch') @@ -60,34 +124,15 @@ class FirefoxUITests(VCSToolsScript, VirtualenvMixin): 'Please specify --firefox-ui-branch. Valid values can be found ' 'in here https://github.com/mozilla/firefox-ui-tests/branches') - def query_abs_dirs(self): - if self.abs_dirs: - return self.abs_dirs - abs_dirs = super(FirefoxUITests, self).query_abs_dirs() + self.installer_url = self.config.get('installer_url') + self.installer_path = self.config.get('installer_path') - dirs = { - 'fx_ui_dir': os.path.join(abs_dirs['abs_work_dir'], 'firefox_ui_tests'), - } - - abs_dirs.update(dirs) - self.abs_dirs = abs_dirs - return self.abs_dirs - - - def checkout(self): - ''' - We checkout firefox_ui_tests and update to the right branch - for it. - ''' - dirs = self.query_abs_dirs() - - self.vcs_checkout( - repo=self.firefox_ui_repo, - dest=dirs['fx_ui_dir'], - branch=self.firefox_ui_branch, - vcs='gittool' - ) + if self.installer_path: + self.installer_path = os.path.abspath(self.installer_path) + if not os.path.exists(self.installer_path): + self.critical('Please make sure that the path to the installer exists.') + sys.exit(1) @PreScriptAction('create-virtualenv') def _pre_create_virtualenv(self, action): @@ -98,6 +143,160 @@ class FirefoxUITests(VCSToolsScript, VirtualenvMixin): url=dirs['fx_ui_dir'], ) + def _query_symbols_url(self, installer_url): + for suffix in INSTALLER_SUFFIXES: + if installer_url.endswith(suffix): + symbols_url = installer_url[:-len(suffix)] + '.crashreporter-symbols.zip' + continue + + if symbols_url: + self.info('Symbols_url: {}'.format(symbols_url)) + if not symbols_url.startswith('http'): + return symbols_url + + try: + # Let's see if the symbols are available + urllib2.urlopen(symbols_url) + return symbols_url + + except urllib2.HTTPError, e: + self.warning('{} - {}'.format(str(e), symbols_url)) + return None + else: + self.fatal('Can\'t find symbols_url from installer_url: {}!'.format(installer_url)) + + def checkout(self): + """ + We checkout firefox_ui_tests and update to the right branch + for it. + """ + dirs = self.query_abs_dirs() + + self.vcs_checkout( + repo=self.firefox_ui_repo, + dest=dirs['fx_ui_dir'], + branch=self.firefox_ui_branch, + vcs='gittool' + ) + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + + abs_dirs = VCSToolsScript.query_abs_dirs(self) + abs_dirs.update({ + 'fx_ui_dir': os.path.join(abs_dirs['abs_work_dir'], 'firefox_ui_tests'), + }) + self.abs_dirs = abs_dirs + + return self.abs_dirs + + def query_extra_cmd_args(self): + """Collects specific update test related command line arguments. + + Sub classes should override this method for their own specific arguments. + """ + return [] + + def run_test(self, installer_path, script_name, env=None, symbols_url=None, + cleanup=True, marionette_port=2828): + """All required steps for running the tests against an installer.""" + dirs = self.query_abs_dirs() + + bin_dir = os.path.dirname(self.query_python_path()) + fx_ui_tests_bin = os.path.join(bin_dir, script_name) + gecko_log = os.path.join(dirs['abs_log_dir'], 'gecko.log') + + # Build the command + cmd = [ + fx_ui_tests_bin, + '--installer', installer_path, + # Log to stdout until tests are stable. + '--gecko-log=-', + '--address', 'localhost:{}'.format(marionette_port), + # Use the work dir to get temporary data stored + '--workspace', dirs['abs_work_dir'], + ] + + if symbols_url: + cmd += ['--symbols-path', symbols_url] + + # Collect all pass-through harness options to the script + cmd.extend(self.query_extra_cmd_args()) + + return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], + output_timeout=300, env=env) + + # Return more output if we fail + if return_code: + if os.path.exists(gecko_log): + contents = self.read_from_file(gecko_log, verbose=False) + self.warning('== Dumping gecko output ==') + self.warning(contents) + self.warning('== End of gecko output ==') + else: + # We're outputting to stdout with --gecko-log=- so there is not log to + # complaing about. Remove the commented line below when changing + # this behaviour. + # self.warning('No gecko.log was found: %s' % gecko_log) + pass + + if cleanup: + for filepath in (installer_path, gecko_log): + if os.path.exists(filepath): + self.debug('Removing {}'.format(filepath)) + os.remove(filepath) + + return return_code + + @PreScriptAction('run-tests') + def _pre_run_tests(self, action): + if not self.installer_path and not self.installer_url: + self.critical('Please specify an installer via --installer-path or --installer-url.') + sys.exit(1) def run_tests(self): - pass + dirs = self.query_abs_dirs() + + if self.installer_url: + self.installer_path = self.download_file( + self.installer_url, + parent_dir=dirs['abs_work_dir'] + ) + + symbols_url = self._query_symbols_url(installer_url=self.installer_path) + + return self.run_test( + installer_path=self.installer_path, + script_name=self.cli_script, + env=self.query_env(), + symbols_url=symbols_url, + cleanup=False, + ) + + +class FirefoxUIUpdateTests(FirefoxUITests): + + cli_script = 'firefox-ui-update' + + def __init__(self, config_options=None, *args, **kwargs): + config_options = config_options or firefox_ui_update_config_options + + FirefoxUITests.__init__(self, config_options=config_options, + *args, **kwargs) + + def query_extra_cmd_args(self): + """Collects specific update test related command line arguments.""" + args = [] + + for option in firefox_ui_update_harness_config_options: + dest = option[1]['dest'] + name = self.config.get(dest) + + if name: + if type(name) is bool: + args.append(option[0][0]) + else: + args.extend([option[0][0], self.config[dest]]) + + return args diff --git a/testing/mozharness/scripts/firefox_ui_tests/update.py b/testing/mozharness/scripts/firefox_ui_tests/update.py new file mode 100755 index 00000000000..9396a722b45 --- /dev/null +++ b/testing/mozharness/scripts/firefox_ui_tests/update.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""firefox_ui_updates.py + +Author: Armen Zambrano G. + Henrik Skupin +""" +import os +import sys + +# load modules from parent dir +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUIUpdateTests + + +if __name__ == '__main__': + myScript = FirefoxUIUpdateTests() + myScript.run_and_exit() diff --git a/testing/mozharness/scripts/firefox_ui_tests/update_release.py b/testing/mozharness/scripts/firefox_ui_tests/update_release.py new file mode 100755 index 00000000000..a2d364b8781 --- /dev/null +++ b/testing/mozharness/scripts/firefox_ui_tests/update_release.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +"""firefox_ui_updates.py + +Author: Armen Zambrano G. + Henrik Skupin +""" +import copy +import os +import pprint +import re +import sys +import urllib + +# load modules from parent dir +sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) + +from mozharness.base.python import PreScriptAction +from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, EXIT_STATUS_DICT +from mozharness.mozilla.testing.firefox_ui_tests import ( + FirefoxUIUpdateTests, + firefox_ui_update_config_options +) + + +# Command line arguments for release update tests +firefox_ui_update_release_config_options = [ + [['--build-number'], { + 'dest': 'build_number', + 'help': 'Build number of release, eg: 2', + }], + [['--limit-locales'], { + 'dest': 'limit_locales', + 'default': -1, + 'type': int, + 'help': 'Limit the number of locales to run.', + }], + [['--release-update-config'], { + 'dest': 'release_update_config', + 'help': 'Name of the release update verification config file to use.', + }], + [['--this-chunk'], { + 'dest': 'this_chunk', + 'default': 1, + 'help': 'What chunk of locales to process.', + }], + [['--tools-repo'], { + 'dest': 'tools_repo', + 'default': 'http://hg.mozilla.org/build/tools', + 'help': 'Which tools repo to check out', + }], + [['--tools-tag'], { + 'dest': 'tools_tag', + 'help': 'Which revision/tag to use for the tools repository.', + }], + [['--total-chunks'], { + 'dest': 'total_chunks', + 'default': 1, + 'help': 'Total chunks to dive the locales into.', + }], +] + copy.deepcopy(firefox_ui_update_config_options) + + +class ReleaseFirefoxUIUpdateTests(FirefoxUIUpdateTests): + + def __init__(self): + all_actions = [ + 'clobber', + 'checkout', + 'create-virtualenv', + 'read-release-update-config', + 'run-tests', + ] + + FirefoxUIUpdateTests.__init__(self, all_actions=all_actions, + default_actions=all_actions, + config_options=firefox_ui_update_release_config_options, + append_env_variables_from_configs=True) + + self.tools_repo = self.config.get('tools_repo') + self.tools_tag = self.config.get('tools_tag') + + assert self.tools_repo and self.tools_tag, \ + 'Without the "--tools-tag" we can\'t clone the releng\'s tools repository.' + + self.limit_locales = int(self.config.get('limit_locales')) + + # This will be a list containing one item per release based on configs + # from tools/release/updates/*cfg + self.releases = None + + def _modify_url(self, rel_info): + # This is a temporary hack to find crash symbols. It should be replaced + # with something that doesn't make wild guesses about where symbol + # packages are. + # We want this: + # https://ftp.mozilla.org/pub/mozilla.org/firefox/candidates/40.0b1-candidates/build1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip + # https://ftp.mozilla.org/pub/mozilla.org//firefox/releases/40.0b1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip + installer_from = rel_info['from'] + version = (re.search('/firefox/releases/(%s.*)\/.*\/.*\/.*' % rel_info['release'], + installer_from)).group(1) + + temp_from = installer_from.replace(version, '%s-candidates/build%s' % ( + version, self.config['build_number']), + 1).replace('releases', 'candidates') + temp_url = rel_info['ftp_server_from'] + urllib.quote(temp_from.replace('%locale%', + 'en-US')) + self.info('Installer url under stage/candidates dir: {}'.format(temp_url)) + + return temp_url + + def checkout(self): + """ + We checkout the tools repository and update to the right branch + for it. + """ + dirs = self.query_abs_dirs() + + FirefoxUIUpdateTests.checkout(self) + + self.vcs_checkout( + repo=self.tools_repo, + dest=dirs['abs_tools_dir'], + revision=self.tools_tag, + vcs='hgtool' + ) + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + + abs_dirs = FirefoxUIUpdateTests.query_abs_dirs(self) + abs_dirs.update({ + 'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'), + }) + self.abs_dirs = abs_dirs + + return self.abs_dirs + + def read_release_update_config(self): + ''' + Builds a testing matrix based on an update verification configuration + file under the tools repository (release/updates/*.cfg). + + Each release info line of the update verification files look similar to the following. + + NOTE: This shows each pair of information as a new line but in reality + there is one white space separting them. We only show the values we care for. + + release="38.0" + platform="Linux_x86_64-gcc3" + build_id="20150429135941" + locales="ach af ... zh-TW" + channel="beta-localtest" + from="/firefox/releases/38.0b9/linux-x86_64/%locale%/firefox-38.0b9.tar.bz2" + ftp_server_from="http://stage.mozilla.org/pub/mozilla.org" + + We will store this information in self.releases as a list of releases. + + NOTE: We will talk of full and quick releases. Full release info normally contains a subset + of all locales (except for the most recent releases). A quick release has all locales, + however, it misses the fields 'from' and 'ftp_server_from'. + Both pairs of information complement each other but differ in such manner. + ''' + dirs = self.query_abs_dirs() + assert os.path.exists(dirs['abs_tools_dir']), \ + 'Without the tools/ checkout we can\'t use releng\'s config parser.' + + if self.config.get('release_update_config'): + # The config file is part of the tools repository. Make sure that if specified + # we force a revision of that repository to be set. + if self.tools_tag is None: + self.fatal('Make sure to specify the --tools-tag') + + self.release_update_config = self.config['release_update_config'] + + # Import the config parser + sys.path.insert(1, os.path.join(dirs['abs_tools_dir'], 'lib', 'python')) + from release.updates.verify import UpdateVerifyConfig + + uvc = UpdateVerifyConfig() + config_file = os.path.join(dirs['abs_tools_dir'], 'release', 'updates', + self.config['release_update_config']) + uvc.read(config_file) + if not hasattr(self, 'update_channel'): + self.update_channel = uvc.channel + + # Filter out any releases that are less than Gecko 38 + uvc.releases = [r for r in uvc.releases + if int(r['release'].split('.')[0]) >= 38] + + temp_releases = [] + for rel_info in uvc.releases: + # This is the full release info + if 'from' in rel_info and rel_info['from'] is not None: + # Let's find the associated quick release which contains the remaining locales + # for all releases except for the most recent release which contain all locales + quick_release = uvc.getRelease(build_id=rel_info['build_id'], from_path=None) + if quick_release != {}: + rel_info['locales'] = sorted(rel_info['locales'] + quick_release['locales']) + temp_releases.append(rel_info) + + uvc.releases = temp_releases + chunked_config = uvc.getChunk( + chunks=int(self.config['total_chunks']), + thisChunk=int(self.config['this_chunk']) + ) + + self.releases = chunked_config.releases + + @PreScriptAction('run-tests') + def _pre_run_tests(self, action): + assert 'release_update_config' in self.config, \ + 'You have to specify --release-update-config.' + + def run_tests(self): + dirs = self.query_abs_dirs() + + # We don't want multiple outputs of the same environment information. To prevent + # that, we can't make it an argument of run_command and have to print it on our own. + self.info('Using env: {}'.format(pprint.pformat(self.query_env()))) + + results = {} + + locales_counter = 0 + for rel_info in sorted(self.releases, key=lambda release: release['build_id']): + build_id = rel_info['build_id'] + results[build_id] = {} + + self.info('About to run {buildid} {path} - {num_locales} locales'.format( + buildid=build_id, + path=rel_info['from'], + num_locales=len(rel_info['locales']) + )) + + # Each locale gets a fresh port to avoid address in use errors in case of + # tests that time out unexpectedly. + marionette_port = 2827 + for locale in rel_info['locales']: + locales_counter += 1 + self.info('Running {buildid} {locale}'.format(buildid=build_id, + locale=locale)) + + if self.limit_locales > -1 and locales_counter > self.limit_locales: + self.info('We have reached the limit of locales we were intending to run') + break + + if self.config['dry_run']: + continue + + # Safe temporary hack to determine symbols URL from en-US + # build1 in the candidates dir + ftp_candidates_installer_url = self._modify_url(rel_info) + symbols_url = self._query_symbols_url( + installer_url=ftp_candidates_installer_url) + + # Determine from where to download the file + installer_url = '{server}/{fragment}'.format( + server=rel_info['ftp_server_from'], + fragment=urllib.quote(rel_info['from'].replace('%locale%', locale)) + ) + installer_path = self.download_file( + url=installer_url, + parent_dir=dirs['abs_work_dir'] + ) + + marionette_port += 1 + + retcode = self.run_test( + installer_path=installer_path, + script_name=self.cli_script, + env=self.query_env(avoid_host_env=True), + symbols_url=symbols_url, + marionette_port=marionette_port, + ) + + if retcode: + self.warning('FAIL: {} has failed.'.format(sys.argv[0])) + + base_cmd = 'python {command} --firefox-ui-branch {branch} ' \ + '--release-update-config {config} --tools-tag {tag}'.format( + command=sys.argv[0], + branch=self.firefox_ui_branch, + config=self.release_update_config, + tag=self.tools_tag + ) + + for config in self.config['config_files']: + base_cmd += ' --cfg {}'.format(config) + + if symbols_url: + base_cmd += ' --symbols-path {}'.format(symbols_url) + + base_cmd += ' --installer-url {}'.format(installer_url) + + self.info('You can run the *specific* locale on the same machine with:') + self.info(base_cmd) + + self.info('You can run the *specific* locale on *your* machine with:') + self.info('{} --cfg developer_config.py'.format(base_cmd)) + + results[build_id][locale] = retcode + + self.info('Completed {buildid} {locale} with return code: {retcode}'.format( + buildid=build_id, + locale=locale, + retcode=retcode)) + + if self.limit_locales > -1 and locales_counter > self.limit_locales: + break + + # Determine which locales have failed and set scripts exit code + exit_status = TBPL_SUCCESS + for build_id in sorted(results.keys()): + failed_locales = [] + for locale in sorted(results[build_id].keys()): + if results[build_id][locale] != 0: + failed_locales.append(locale) + + if failed_locales: + if exit_status == TBPL_SUCCESS: + self.info('\nSUMMARY - Failed locales for {}:'.format(self.cli_script)) + self.info('====================================================') + exit_status = TBPL_WARNING + + self.info(build_id) + self.info(' {}'.format(', '.join(failed_locales))) + + self.return_code = EXIT_STATUS_DICT[exit_status] + + +if __name__ == '__main__': + myScript = ReleaseFirefoxUIUpdateTests() + myScript.run_and_exit() diff --git a/testing/mozharness/scripts/firefox_ui_updates.py b/testing/mozharness/scripts/firefox_ui_updates.py deleted file mode 100755 index 5133232d1a8..00000000000 --- a/testing/mozharness/scripts/firefox_ui_updates.py +++ /dev/null @@ -1,475 +0,0 @@ -#!/usr/bin/env python -# ***** BEGIN LICENSE BLOCK ***** -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. -# ***** END LICENSE BLOCK ***** -"""firefox_ui_updates.py - -Author: Armen Zambrano G. -""" -import copy -import os -import re -import urllib -import urllib2 -import sys - -# load modules from parent dir -sys.path.insert(1, os.path.dirname(sys.path[0])) - -from mozharness.base.script import PreScriptAction -from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUITests -from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, EXIT_STATUS_DICT - -INSTALLER_SUFFIXES = ('.tar.bz2', '.zip', '.dmg', '.exe', '.apk', '.tar.gz') - - -class FirefoxUIUpdates(FirefoxUITests): - # This will be a list containing one item per release based on configs - # from tools/release/updates/*cfg - releases = None - channel = None - harness_extra_args = [ - [['--update-allow-mar-channel'], { - 'dest': 'update_allow_mar_channel', - 'help': 'Additional MAR channel to be allowed for updates, e.g. ' - '"firefox-mozilla-beta" for updating a release build to ' - 'the latest beta build.', - }], - [['--update-channel'], { - 'dest': 'update_channel', - 'help': 'Update channel to use.', - }], - [['--update-target-version'], { - 'dest': 'update_target_version', - 'help': 'Version of the updated build.', - }], - [['--update-target-buildid'], { - 'dest': 'update_target_buildid', - 'help': 'Build ID of the updated build', - }], - [['--symbols-path=SYMBOLS_PATH'], { - 'dest': 'symbols_path', - 'help': 'absolute path to directory containing breakpad ' - 'symbols, or the url of a zip file containing symbols.', - }], - ] - - def __init__(self): - config_options = [ - [['--tools-repo'], { - 'dest': 'tools_repo', - 'default': 'http://hg.mozilla.org/build/tools', - 'help': 'Which tools repo to check out', - }], - [['--tools-tag'], { - 'dest': 'tools_tag', - 'help': 'Which revision/tag to use for the tools repository.', - }], - [['--update-verify-config'], { - 'dest': 'update_verify_config', - 'help': 'Which update verify config file to use.', - }], - [['--this-chunk'], { - 'dest': 'this_chunk', - 'default': 1, - 'help': 'What chunk of locales to process.', - }], - [['--total-chunks'], { - 'dest': 'total_chunks', - 'default': 1, - 'help': 'Total chunks to dive the locales into.', - }], - [['--dry-run'], { - 'dest': 'dry_run', - 'default': False, - 'help': 'Only show what was going to be tested.', - }], - [["--build-number"], { - "dest": "build_number", - "help": "Build number of release, eg: 2", - }], - # These are options when we don't use the releng update config file - [['--installer-url'], { - 'dest': 'installer_url', - 'help': 'Point to an installer to download and test against.', - }], - [['--installer-path'], { - 'dest': 'installer_path', - 'help': 'Point to an installer to test against.', - }], - [['--limit-locales'], { - 'dest': 'limit_locales', - 'default': -1, - 'type': int, - 'help': 'Limit the number of locales to run.', - }], - ] + copy.deepcopy(self.harness_extra_args) - - super(FirefoxUIUpdates, self).__init__( - config_options=config_options, - all_actions=[ - 'clobber', - 'checkout', - 'create-virtualenv', - 'determine-testing-configuration', - 'run-tests', - ], - append_env_variables_from_configs=True, - ) - - dirs = self.query_abs_dirs() - - self.limit_locales = int(self.config.get('limit_locales')) - - self.tools_repo = self.config['tools_repo'] - self.tools_tag = self.config.get('tools_tag') - - if self.config.get('update_verify_config'): - self.update_verify_config = self.config['update_verify_config'] - self.updates_config_file = os.path.join( - dirs['abs_tools_dir'], 'release', 'updates', - self.config['update_verify_config'] - ) - - # We want to make sure that anyone trying to reproduce a job will - # is using the exact tools tag for reproducibility's sake - if self.config.get('tools_tag') is None: - self.fatal('Make sure to specify the --tools-tag') - - self.installer_url = self.config.get('installer_url') - self.installer_path = self.config.get('installer_path') - - if self.installer_path: - self.installer_path = os.path.abspath(self.installer_path) - - if not os.path.exists(self.installer_path): - self.critical('Please make sure that the path to the installer exists.') - exit(1) - - assert ('update_verify_config' in self.config or - self.installer_url or self.installer_path, - 'Either specify --update-verify-config, --installer-url or --installer-path.') - - def query_abs_dirs(self): - if self.abs_dirs: - return self.abs_dirs - abs_dirs = super(FirefoxUIUpdates, self).query_abs_dirs() - - dirs = { - 'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'), - } - - abs_dirs.update(dirs) - self.abs_dirs = abs_dirs - return self.abs_dirs - - def checkout(self): - ''' - This checkouts firefox_ui_tests and update to the right branch - for it. - - We also checkout the tools repo if requested because it contains the - configuration files about which locales to test. - - ''' - super(FirefoxUIUpdates, self).checkout() - dirs = self.query_abs_dirs() - - if self.tools_tag: - self.vcs_checkout( - repo=self.tools_repo, - dest=dirs['abs_tools_dir'], - revision=self.tools_tag, - vcs='hgtool' - ) - - def determine_testing_configuration(self): - ''' - This method builds a testing matrix either based on an update verification - configuration file under the tools repo (release/updates/*.cfg) - OR it skips it when we use --installer-url --installer-path - - Each release info line of the update verification files look similar to the following. - - NOTE: This shows each pair of information as a new line but in reality - there is one white space separting them. We only show the values we care for. - - release="38.0" - platform="Linux_x86_64-gcc3" - build_id="20150429135941" - locales="ach af ... zh-TW" - channel="beta-localtest" - from="/firefox/releases/38.0b9/linux-x86_64/%locale%/firefox-38.0b9.tar.bz2" - ftp_server_from="http://stage.mozilla.org/pub/mozilla.org" - - We will store this information in self.releases as a list of releases. - - NOTE: We will talk of full and quick releases. Full release info normally contains a subset - of all locales (except for the most recent releases). A quick release has all locales, - however, it misses the fields 'from' and 'ftp_server_from'. - Both pairs of information complement each other but differ in such manner. - ''' - if self.installer_url or self.installer_path: - return - - dirs = self.query_abs_dirs() - assert os.path.exists(dirs['abs_tools_dir']), \ - "Without the tools/ checkout we can't use releng's config parser." - - # Import the config parser - sys.path.insert(1, os.path.join(dirs['abs_tools_dir'], 'lib', 'python')) - from release.updates.verify import UpdateVerifyConfig - - uvc = UpdateVerifyConfig() - uvc.read(self.updates_config_file) - self.channel = uvc.channel - - # Filter out any releases that are less than Gecko 38 - uvc.releases = [r for r in uvc.releases - if int(r["release"].split('.')[0]) >= 38] - - temp_releases = [] - for rel_info in uvc.releases: - # This is the full release info - if 'from' in rel_info and rel_info['from'] is not None: - # Let's find the associated quick release which contains the remaining locales - # for all releases except for the most recent release which contain all locales - quick_release = uvc.getRelease(build_id=rel_info['build_id'], from_path=None) - if quick_release != {}: - rel_info['locales'] = sorted(rel_info['locales'] + quick_release['locales']) - temp_releases.append(rel_info) - - uvc.releases = temp_releases - chunked_config = uvc.getChunk( - chunks=int(self.config['total_chunks']), - thisChunk=int(self.config['this_chunk']) - ) - - self.releases = chunked_config.releases - - def _modify_url(self, rel_info): - # This is a temporary hack to find crash symbols. It should be replaced - # with something that doesn't make wild guesses about where symbol - # packages are. - # We want this: - # https://ftp.mozilla.org/pub/mozilla.org/firefox/candidates/40.0b1-candidates/build1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip - # https://ftp.mozilla.org/pub/mozilla.org//firefox/releases/40.0b1/mac/en-US/Firefox%2040.0b1.crashreporter-symbols.zip - installer_from = rel_info['from'] - version = (re.search('/firefox/releases/(%s.*)\/.*\/.*\/.*' % rel_info['release'], installer_from)).group(1) - - temp_from = installer_from.replace(version, '%s-candidates/build%s' % (version, self.config["build_number"]), 1).replace('releases', 'candidates') - temp_url = rel_info["ftp_server_from"] + urllib.quote(temp_from.replace('%locale%', 'en-US')) - self.info('Installer url under stage/candidates dir %s' % temp_url) - - return temp_url - - def _query_symbols_url(self, installer_url): - for suffix in INSTALLER_SUFFIXES: - if installer_url.endswith(suffix): - symbols_url = installer_url[:-len(suffix)] + '.crashreporter-symbols.zip' - continue - - if symbols_url: - self.info('Candidate symbols_url: %s' % symbols_url) - if not symbols_url.startswith('http'): - return symbols_url - - try: - # Let's see if the symbols are available - urllib2.urlopen(symbols_url) - return symbols_url - - except urllib2.HTTPError, e: - self.warning("%s - %s" % (str(e), symbols_url)) - return None - else: - self.fatal("Can't figure out symbols_url from installer_url %s!" % installer_url) - - @PreScriptAction('run-tests') - def _pre_run_tests(self, action): - if self.releases is None and not (self.installer_url or self.installer_path): - self.fatal('You need to call --determine-testing-configuration as well.') - - def _run_test(self, installer_path, symbols_url=None, update_channel=None, cleanup=True, - marionette_port=2828): - ''' - All required steps for running the tests against an installer. - ''' - dirs = self.query_abs_dirs() - env = self.query_env(avoid_host_env=True) - bin_dir = os.path.dirname(self.query_python_path()) - fx_ui_tests_bin = os.path.join(bin_dir, 'firefox-ui-update') - gecko_log = os.path.join(dirs['abs_work_dir'], 'gecko.log') - - # Build the command - cmd = [ - fx_ui_tests_bin, - '--installer', installer_path, - # Log to stdout until tests are stable. - '--gecko-log=-', - '--address=localhost:%s' % marionette_port, - # Use the work dir to get temporary data stored - '--workspace=%s' % dirs['abs_work_dir'], - ] - - if symbols_url: - cmd += ['--symbols-path', symbols_url] - - for arg in self.harness_extra_args: - dest = arg[1]['dest'] - if dest in self.config: - cmd += [' '.join(arg[0]), self.config[dest]] - - if update_channel: - cmd += ['--update-channel', update_channel] - - return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], - output_timeout=300, - env=env) - - # Return more output if we fail - if return_code != 0: - self.info('Internally this is the command fx-ui-updates executed') - self.info('%s' % ' '.join(map(str, cmd))) - - if os.path.exists(gecko_log): - contents = self.read_from_file(gecko_log, verbose=False) - self.warning('== Dumping gecko output ==') - self.warning(contents) - self.warning('== End of gecko output ==') - else: - # We're outputting to stdout with --gecko-log=- so there is not log to - # complaing about. Remove the commented line below when changing - # this behaviour. - # self.warning('No gecko.log was found: %s' % gecko_log) - pass - - if cleanup: - for filepath in (installer_path, gecko_log): - if os.path.exists(filepath): - self.debug('Removing %s' % filepath) - os.remove(filepath) - - return return_code - - def run_tests(self): - dirs = self.query_abs_dirs() - - if self.installer_url or self.installer_path: - if self.installer_url: - self.installer_path = self.download_file( - self.installer_url, - parent_dir=dirs['abs_work_dir'] - ) - - symbols_url = self._query_symbols_url(installer_url=self.installer_path) - - return self._run_test( - installer_path=self.installer_path, - symbols_url=symbols_url, - cleanup=False - ) - - else: - results = {} - - locales_counter = 0 - for rel_info in sorted(self.releases, key=lambda release: release['build_id']): - build_id = rel_info['build_id'] - results[build_id] = {} - - self.info('About to run %s %s - %s locales' % ( - build_id, - rel_info['from'], - len(rel_info['locales']) - )) - - # Each locale gets a fresh port to avoid address in use errors in case of - # tests that time out unexpectedly. - marionette_port = 2827 - for locale in rel_info['locales']: - locales_counter += 1 - self.info("Running %s %s" % (build_id, locale)) - - if self.limit_locales and locales_counter > self.limit_locales: - self.info("We have reached the limit of locales we were intending to run") - break - - if self.config['dry_run']: - continue - - # Safe temp hack to determine symbols URL from en-US build1 in the candidates dir - ftp_candidates_installer_url = self._modify_url(rel_info) - symbols_url = self._query_symbols_url(installer_url=ftp_candidates_installer_url) - - # Determine from where to download the file - installer_url = '%s/%s' % ( - rel_info['ftp_server_from'], - urllib.quote(rel_info['from'].replace('%locale%', locale)) - ) - installer_path = self.download_file( - url=installer_url, - parent_dir=dirs['abs_work_dir'] - ) - - marionette_port += 1 - - retcode = self._run_test( - installer_path=installer_path, - symbols_url=symbols_url, - update_channel=self.channel, - marionette_port=marionette_port) - - if retcode != 0: - self.warning('FAIL: firefox-ui-update has failed.') - - base_cmd = 'python scripts/firefox_ui_updates.py' - for c in self.config['config_files']: - base_cmd += ' --cfg %s' % c - - base_cmd += ' --firefox-ui-branch %s --update-verify-config %s --tools-tag %s' % \ - (self.firefox_ui_branch, self.update_verify_config, self.tools_tag) - - base_cmd += ' --installer-url %s' % installer_url - if symbols_url: - base_cmd += ' --symbols-path %s' % symbols_url - - self.info('You can run the *specific* locale on the same machine with:') - self.info('%s' % base_cmd) - - self.info('You can run the *specific* locale on *your* machine with:') - self.info('%s --cfg developer_config.py' % base_cmd) - - results[build_id][locale] = retcode - - self.info("Completed %s %s with return code: %s" % (build_id, locale, retcode)) - - if self.limit_locales and locales_counter > self.limit_locales: - break - - # Determine which locales have failed and set scripts exit code - exit_status = TBPL_SUCCESS - for build_id in sorted(results.keys()): - failed_locales = [] - for locale in sorted(results[build_id].keys()): - if results[build_id][locale] != 0: - failed_locales.append(locale) - - if failed_locales: - if exit_status == TBPL_SUCCESS: - self.info("") - self.info("SUMMARY - Firefox UI update tests failed locales:") - self.info("=================================================") - exit_status = TBPL_WARNING - - self.info(build_id) - self.info(" %s" % (', '.join(failed_locales))) - - self.return_code = EXIT_STATUS_DICT[exit_status] - - -if __name__ == '__main__': - myScript = FirefoxUIUpdates() - myScript.run_and_exit()