From a34ee420455519a5d98e2e5c65a57b7b70a1c0dd Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Fri, 22 Jan 2016 11:14:11 +0100 Subject: [PATCH] bug 1151954 - move get_apk.sh to a mozharness file r=sylvestre --- testing/mozharness/.hgignore | 25 +++ testing/mozharness/scripts/get_apk.py | 243 ++++++++++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 testing/mozharness/.hgignore create mode 100644 testing/mozharness/scripts/get_apk.py diff --git a/testing/mozharness/.hgignore b/testing/mozharness/.hgignore new file mode 100644 index 00000000000..09872122c46 --- /dev/null +++ b/testing/mozharness/.hgignore @@ -0,0 +1,25 @@ +syntax: glob + +.hgignore +.gitignore +.pydevproject +.project +.tox/ +*.log +*.pyc +*.swp +.DS_Store +.coverage +coverage +htmlcov +nosetests.xml +venv +bin/ +build/ +include/ +lib/ +local/ +logs/ +docs/_build/ +*~ +apk-download diff --git a/testing/mozharness/scripts/get_apk.py b/testing/mozharness/scripts/get_apk.py new file mode 100644 index 00000000000..d6b7fdad363 --- /dev/null +++ b/testing/mozharness/scripts/get_apk.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python + +import sys +import os +import re +import signal + +# load modules from parent dir +sys.path.insert(1, os.path.dirname(sys.path[0])) + +# import the guts +from mozharness.base.script import BaseScript +from mozharness.base.python import VirtualenvMixin +from mozharness.base.script import ScriptMixin + + +class GetAPK(BaseScript, VirtualenvMixin): + all_actions = [ + 'create-virtualenv', + "test", + 'download-apk' + ] + + default_actions = [ + 'create-virtualenv', + 'test' + ] + + config_options = [ + [["--build"], { + "dest": "build", + "help": "Specify build number (default 1)", + "default": "1" + }], + [["--version"], { + "dest": "version", + "help": "Specify version number to download (e.g. 23.0b7)", + "default": "None" + }], + [["--arch"], { + "dest": "arch", + "help": "Specify which architecture to get the apk for", + "default": "all" + }], + [["--locale"], { + "dest": "locale", + "help": "Specify which locale to get the apk for", + "default": "multi" + }], + [["--clean"], { + "dest": "clean", + "help": "Use this option to clean the download directory", + "action": "store_true", + "default": False + }] + ] + + arch_values = ["arm", "x86"] + multi_api_archs = ["arm"] + multi_apis = ["api-9", "api-11"] + + download_dir = "apk-download" + + apk_ext = ".apk" + checksums_ext = ".checksums" + android_prefix = "android-" + + # Cleanup half downloaded files on Ctrl+C + def signal_handler(self, signal, frame): + print("You pressed Ctrl+C!") + self.cleanup() + sys.exit(1) + + def cleanup(self): + ScriptMixin.rmtree(self, self.download_dir) + self.info("Download directory cleaned") + + def __init__(self, require_config_file=False, config={}, + all_actions=all_actions, + default_actions=default_actions): + default_config = { + # the path inside the work_dir ('build') of where we will install the env. + # pretty sure it's the default and not needed. + 'virtualenv_path': 'venv', + } + + default_config.update(config) + + BaseScript.__init__( + self, + config_options=self.config_options, + require_config_file=require_config_file, + config=default_config, + all_actions=all_actions, + ) + + # Gets called once download is complete + def download_complete(self, apk_file, checksum_file): + self.info(apk_file + " has been downloaded successfully") + ScriptMixin.rmtree(self, checksum_file) + + # Called if download fails due to 404 error or some other connection failure + def download_error(self): + self.cleanup() + self.fatal("Download failed!") + + # Check the given values are correct + def check_argument(self): + if self.config["clean"]: + self.cleanup() + if self.config["version"] == "None": + if self.config["clean"]: + sys.exit(0) + self.fatal("Version is required") + + if self.config["arch"] not in self.arch_values and not self.config["arch"] == "all": + error = self.config["arch"] + " is not a valid arch. " \ + "Try one of the following:"+os.linesep + for arch in self.arch_values: + error += arch + os.linesep + error += "Or don't use the --arch option to download all the archs" + self.fatal(error) + + # Checksum check the APK + def check_apk(self, apk_file, checksum_file): + self.info("The checksum for the APK is being checked....") + checksum = ScriptMixin.read_from_file(self, checksum_file, False) + checksum = re.sub("\s(.*)", "", checksum.splitlines()[0]) + + apk_checksum = self.file_sha512sum(apk_file) + + if checksum == apk_checksum: + self.info("APK checksum check succeeded!") + self.download_complete(apk_file, checksum_file) + else: + ScriptMixin.rmtree(self, self.download_dir) + self.fatal("Downloading " + apk_file + " failed!") + + # Helper functions + def generate_url(self, version, build, locale, api_suffix, arch_file): + return "https://ftp.mozilla.org/pub/mozilla.org/mobile/candidates/" + version + "-candidates/build" + build + \ + "/" + self.android_prefix + api_suffix + "/" + locale + "/fennec-" + version + "." + locale + \ + "." + self.android_prefix + arch_file + + def get_api_suffix(self, arch): + if arch in self.multi_api_archs: + return self.multi_apis + else: + return [arch] + + def get_arch_file(self, arch): + if arch == "x86": + # the filename contains i386 instead of x86 + return "i386" + else: + return arch + + def get_common_file_name(self, version, locale): + return "fennec-" + version + "." + locale + "." + self.android_prefix + + # Function that actually downloads the file + def download(self, version, build, arch, locale): + ScriptMixin.mkdir_p(self, self.download_dir) + + common_filename = self.get_common_file_name(version, locale) + arch_file = self.get_arch_file(arch) + + for api_suffix in self.get_api_suffix(arch): + url = self.generate_url(version, build, locale, api_suffix, arch_file) + apk_url = url + self.apk_ext + checksum_url = url + self.checksums_ext + if arch in self.multi_api_archs: + filename = common_filename + arch_file + "-" + api_suffix + else: + filename = common_filename + arch_file + + filename_apk = os.path.join(self.download_dir, filename + self.apk_ext) + filename_checksums = os.path.join(self.download_dir, filename + self.checksums_ext) + + retry_config = {'attempts': 1, 'cleanup': self.download_error} + ScriptMixin.download_file(self, apk_url, filename_apk, retry_config=retry_config) + + retry_config = {'attempts': 1, 'cleanup': self.download_error} + ScriptMixin.download_file(self, checksum_url, filename_checksums, retry_config=retry_config) + + self.check_apk(filename_apk, filename_checksums) + + # Download all the archs if none is given + def download_all(self, version, build, locale): + for arch in self.arch_values: + self.download(version, build, arch, locale) + + # Download apk initial action + def download_apk(self): + self.check_argument() + version = self.config["version"] + arch = self.config["arch"] + build = str(self.config["build"]) + locale = self.config["locale"] + + self.info("Downloading version " + version + " build #" + build + + " for arch " + arch + " (locale " + locale + ")") + if arch == "all": + self.download_all(version, build, locale) + else: + self.download(version, build, arch, locale) + + # Test the helper, cleanup and check functions + def test(self): + ScriptMixin.mkdir_p(self, self.download_dir) + testfile = os.path.join(self.download_dir, "testfile") + testchecksums = os.path.join(self.download_dir, "testchecksums") + ScriptMixin.write_to_file(self, testfile, "This is a test file!") + ScriptMixin.write_to_file(self, testchecksums, self.file_sha512sum(testfile)) + self.check_apk(testfile, testchecksums) + self.download_complete(testfile, testchecksums) + if os.path.isfile(testchecksums): + self.fatal("download_complete test failed!") + + self.cleanup() + if os.path.isdir(self.download_dir): + self.fatal("cleanup test failed") + + url = self.generate_url("43.0", "2", "multi", "x86", "i386") + correcturl = "https://ftp.mozilla.org/pub/mozilla.org/mobile/candidates/43.0-candidates/build2/"\ + + self.android_prefix + "x86/multi/fennec-43.0.multi." + self.android_prefix + "i386" + if not url == correcturl: + self.fatal("get_url test failed!") + + if not self.get_api_suffix(self.multi_api_archs[0]) == self.multi_apis: + self.fatal("get_api_suffix test failed!") + + if not self.get_arch_file("x86") == "i386": + self.fatal("get_arch_file test failed!") + + if not self.get_common_file_name("43.0", "multi") == "fennec-43.0.multi." + self.android_prefix: + self.fatal("get_common_file_name test failed!") + +# main {{{1 +if __name__ == '__main__': + myScript = GetAPK() + signal.signal(signal.SIGINT, myScript.signal_handler) + myScript.run_and_exit()