wine/gecko
0
0
Fork 0
mirror of https://gitlab.winehq.org/wine/wine-gecko.git synced 2024-09-13 09:24:08 -07:00
Code Issues Packages Projects Releases Wiki Activity

Bug 1237706 - Use general unpack() method to extract all kinds of supported archives. r=jlund

Browse Source
This commit is contained in:
Henrik Skupin 2016-01-13 23:14:09 +01:00
parent 7fe5eea4b7
commit 9a8d9b711b
11 changed files with 79 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
testing/mozharness/mozharness/base/script.py
View File

@ -45,10 +45,11 @@ except ImportError:
from mozprocess import ProcessHandler
from mozharness.base.config import BaseConfig
from mozharness.base.errors import ZipErrorList
from mozharness.base.errors import TarErrorList, ZipErrorList
from mozharness.base.log import SimpleFileLogger, MultiFileLogger, \
LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL
def platform_name():
pm = PlatformMixin()
@ -451,7 +452,7 @@ class ScriptMixin(PlatformMixin):
**retry_args
)
def download_unzip(self, url, parent_dir, target_unzip_dirs=None, halt_on_failure=True):
def download_unpack(self, url, parent_dir, extract_dirs=None, halt_on_failure=True):
"""Generic method to download and extract a zip file.
The downloaded file will always be saved to the working directory and is not getting
@ -461,29 +462,16 @@ class ScriptMixin(PlatformMixin):
url (str): URL where the file to be downloaded is located.
parent_dir (str): directory where the downloaded file will
be extracted to.
target_unzip_dirs (list, optional): directories inside the zip file to extract.
extract_dirs (list, optional): directories inside the archive file to extract.
Defaults to `None`.
halt_on_failure (bool, optional): whether or not to redefine the
log level as `FATAL` on errors. Defaults to True.
"""
dirs = self.query_abs_dirs()
zipfile = self.download_file(url, parent_dir=dirs['abs_work_dir'],
archive = self.download_file(url, parent_dir=dirs['abs_work_dir'],
error_level=FATAL)
command = self.query_exe('unzip', return_type='list')
# Always overwrite to not get an input in a hidden pipe if files already exist
command.extend(['-q', '-o', zipfile, '-d', parent_dir])
if target_unzip_dirs:
command.extend(target_unzip_dirs)
# TODO error_list: http://www.info-zip.org/mans/unzip.html#DIAGNOSTICS
# unzip return code 11 is 'no matching files were found'
self.run_command(command,
error_list=ZipErrorList,
halt_on_failure=halt_on_failure,
fatal_exit_code=3,
success_codes=[0, 11],
)
self.unpack(archive, parent_dir, extract_dirs, halt_on_failure)
def load_json_url(self, url, error_level=None, *args, **kwargs):
""" Returns a json object from a url (it retries). """
@ -1395,26 +1383,55 @@ class ScriptMixin(PlatformMixin):
self.log(msg, error_level=error_level)
os.utime(file_name, times)
def unpack(self, filename, extract_to):
'''
def unpack(self, filename, extract_to,
extract_dirs=None, halt_on_failure=True):
"""
This method allows us to extract a file regardless of its extension
Args:
filename (str): filename of the compressed file.
extract_to (str): where to extract the compressed file.
'''
# XXX: Make sure that filename has a extension of one of our supported file formats
extract_dirs (list, optional): directories inside the archive file to extract.
Defaults to `None`.
halt_on_failure (bool, optional): whether or not to redefine the
log level as `FATAL` on errors. Defaults to True.
"""
m = re.search('\.tar\.(bz2|gz)$', filename)
if m:
# tar does not automatically create the target folder
if not os.path.exists(extract_to):
os.makedirs(extract_to)
command = self.query_exe('tar', return_type='list')
tar_cmd = "jxfv"
if m.group(1) == "gz":
tar_cmd = "zxfv"
command.extend([tar_cmd, filename, "-C", extract_to])
self.run_command(command, halt_on_failure=True)
tar_args = 'zxfv' if m.group(1) == 'gz' else 'jxfv'
command.extend([tar_args, filename, '-C', extract_to])
if extract_dirs:
command.append('--wildcards')
command.extend(extract_dirs)
self.run_command(command,
error_list=TarErrorList,
fatal_exit_code=3,
halt_on_failure=halt_on_failure,
)
elif filename.endswith('.zip'):
command = self.query_exe('unzip', return_type='list')
# Always overwrite to not get an input in a hidden pipe if files already exist
command.extend(['-q', '-o', filename, '-d', extract_to])
if extract_dirs:
command.extend(extract_dirs)
# TODO error_list: http://www.info-zip.org/mans/unzip.html#DIAGNOSTICS
# unzip return code 11 is 'no matching files were found'
self.run_command(command,
error_list=ZipErrorList,
fatal_exit_code=3,
halt_on_failure=halt_on_failure,
success_codes=[0, 11],
)
else:
# XXX implement
pass
raise NotImplementedError('No extraction method found for: %s' % filename)
def PreScriptRun(func):

4
testing/mozharness/mozharness/mozilla/testing/talos.py
View File

@ -274,13 +274,13 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
if self.query_pagesets_url():
self.info("Downloading pageset...")
src_talos_pageset = os.path.join(src_talos_webdir, 'tests')
self.download_unzip(self.pagesets_url, src_talos_pageset)
self.download_unpack(self.pagesets_url, src_talos_pageset)
# Action methods. {{{1
# clobber defined in BaseScript
# read_buildbot_config defined in BuildbotMixin
def download_and_extract(self, target_unzip_dirs=None, suite_categories=None):
def download_and_extract(self, extract_dirs=None, suite_categories=None):
return super(Talos, self).download_and_extract(
suite_categories=['common', 'talos']
)

24
testing/mozharness/mozharness/mozilla/testing/testbase.py
View File

@ -396,7 +396,7 @@ You can set this by:
pprint.pformat(package_requirements))
return package_requirements
def _download_test_packages(self, suite_categories, target_unzip_dirs):
def _download_test_packages(self, suite_categories, extract_dirs):
# Some platforms define more suite categories/names than others.
# This is a difference in the convention of the configs more than
# to how these tests are run, so we pave over these differences here.
@ -430,21 +430,21 @@ You can set this by:
(target_packages, category))
for file_name in target_packages:
target_dir = test_install_dir
unzip_dirs = target_unzip_dirs
unpack_dirs = extract_dirs
if "jsshell-" in file_name or file_name == "target.jsshell.zip":
self.info("Special-casing the jsshell zip file")
unzip_dirs = None
unpack_dirs = None
target_dir = dirs['abs_test_bin_dir']
url = self.query_build_dir_url(file_name)
self.download_unzip(url, target_dir,
target_unzip_dirs=unzip_dirs)
self.download_unpack(url, target_dir,
extract_dirs=unpack_dirs)
def _download_test_zip(self, target_unzip_dirs=None):
def _download_test_zip(self, extract_dirs=None):
dirs = self.query_abs_dirs()
test_install_dir = dirs.get('abs_test_install_dir',
os.path.join(dirs['abs_work_dir'], 'tests'))
self.download_unzip(self.test_url, test_install_dir,
target_unzip_dirs=target_unzip_dirs)
self.download_unpack(self.test_url, test_install_dir,
extract_dirs=extract_dirs)
def structured_output(self, suite_category):
"""Defines whether structured logging is in use in this configuration. This
@ -491,9 +491,9 @@ You can set this by:
self.set_buildbot_property("symbols_url", self.symbols_url,
write_to_file=True)
self.download_unzip(self.symbols_url, self.symbols_path)
self.download_unpack(self.symbols_url, self.symbols_path)
def download_and_extract(self, target_unzip_dirs=None, suite_categories=None):
def download_and_extract(self, extract_dirs=None, suite_categories=None):
"""
download and extract test zip / download installer
"""
@ -516,7 +516,7 @@ You can set this by:
' package data at "%s" will be ignored.' %
(self.config.get('test_url'), self.test_packages_url))
self._download_test_zip(target_unzip_dirs)
self._download_test_zip(extract_dirs)
else:
if not self.test_packages_url:
# The caller intends to download harness specific packages, but doesn't know
@ -526,7 +526,7 @@ You can set this by:
self.test_packages_url = self.query_build_dir_url('test_packages.json')
suite_categories = suite_categories or ['common']
self._download_test_packages(suite_categories, target_unzip_dirs)
self._download_test_packages(suite_categories, extract_dirs)
self._download_installer()
if self.config.get('download_symbols'):

2
testing/mozharness/mozharness/mozilla/testing/unittest.py
View File

@ -221,7 +221,7 @@ class EmulatorMixin(object):
dirs = self.query_abs_dirs()
self.mkdir_p(dirs['abs_emulator_dir'])
if self.config.get('emulator_url'):
self.download_unzip(self.config['emulator_url'], dirs['abs_emulator_dir'])
self.download_unpack(self.config['emulator_url'], dirs['abs_emulator_dir'])
elif self.config.get('emulator_manifest'):
manifest_path = self.create_tooltool_manifest(self.config['emulator_manifest'])
do_unzip = True

2
testing/mozharness/scripts/android_emulator_unittest.py
View File

@ -627,7 +627,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
robocop_url = self.installer_url[:self.installer_url.rfind('/')] + '/robocop.apk'
self.info("Downloading robocop...")
self.download_file(robocop_url, 'robocop.apk', dirs['abs_work_dir'], error_level=FATAL)
self.download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
self.download_unpack(self.host_utils_url, dirs['abs_xre_dir'])
def install(self):
"""

2
testing/mozharness/scripts/android_panda.py
View File

@ -383,7 +383,7 @@ class PandaTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixin, Bu
dirs = self.query_abs_dirs()
self.host_utils_url = c['hostutils_url']
# get the zip and extract it
self.download_unzip(self.host_utils_url, dirs['abs_hostutils_dir'])
self.download_unpack(self.host_utils_url, dirs['abs_hostutils_dir'])
def _install_app(self):
c = self.config

6
testing/mozharness/scripts/android_panda_talos.py
View File

@ -299,7 +299,7 @@ class PandaTalosTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixi
self.rmtree(dirs['abs_talosdata_dir'])
self.mkdir_p(dirs['abs_talosdata_dir'])
self.mkdir_p(dirs['abs_symbols_dir'])
self.download_unzip(self.installer_url,
self.download_unpack(self.installer_url,
dirs['abs_fennec_dir'])
#this is ugly but you can't specify a file in download_unzip to extract the file to, by default it's the abs_work_dir
#should think of a better way
@ -317,10 +317,10 @@ class PandaTalosTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixi
error_level=FATAL)
self.symbols_url = self.query_symbols_url()
self.download_unzip(self.symbols_url,
self.download_unpack(self.symbols_url,
dirs['abs_symbols_dir'])
self.download_unzip(self.config['retry_url'],
self.download_unpack(self.config['retry_url'],
dirs['abs_talosdata_dir'])
taloscode = self.config.get("talos_from_code_url")

2
testing/mozharness/scripts/androidx86_emulator_unittest.py
View File

@ -681,7 +681,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
self._download_robocop_apk()
break
self.download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
self.download_unpack(self.host_utils_url, dirs['abs_xre_dir'])
def install(self):
assert self.installer_path is not None, \

2
testing/mozharness/scripts/b2g_emulator_unittest.py
View File

@ -208,7 +208,7 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
error_list=TarErrorList,
halt_on_failure=True, fatal_exit_code=3)
self.download_unzip(self.config['xre_url'],
self.download_unpack(self.config['xre_url'],
dirs['abs_xre_dir'])
if self.config.get('busybox_url'):

8
testing/mozharness/scripts/desktop_unittest.py
View File

@ -475,20 +475,20 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
"""
c = self.config
target_unzip_dirs = None
extract_dirs = None
if c['specific_tests_zip_dirs']:
target_unzip_dirs = list(c['minimum_tests_zip_dirs'])
extract_dirs = list(c['minimum_tests_zip_dirs'])
for category in c['specific_tests_zip_dirs'].keys():
if c['run_all_suites'] or self._query_specified_suites(category) \
or 'run-tests' not in self.actions:
target_unzip_dirs.extend(c['specific_tests_zip_dirs'][category])
extract_dirs.extend(c['specific_tests_zip_dirs'][category])
if c.get('run_all_suites'):
target_categories = SUITE_CATEGORIES
else:
target_categories = [cat for cat in SUITE_CATEGORIES
if self._query_specified_suites(cat) is not None]
super(DesktopUnittest, self).download_and_extract(target_unzip_dirs=target_unzip_dirs,
super(DesktopUnittest, self).download_and_extract(extract_dirs=extract_dirs,
suite_categories=target_categories)
# pull defined in VCSScript.

2
testing/mozharness/scripts/web_platform_tests.py
View File

@ -165,7 +165,7 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin):
def download_and_extract(self):
super(WebPlatformTest, self).download_and_extract(
target_unzip_dirs=["bin/*",
extract_dirs=["bin/*",
"config/*",
"mozbase/*",
"marionette/*",