Bug 1237706 - Use general unpack() method to extract all kinds of supported archives. r=jlund

testing/mozharness/mozharness/base/script.py

@@ -45,10 +45,11 @@ except ImportError:
 
 from mozprocess import ProcessHandler
 from mozharness.base.config import BaseConfig
-from mozharness.base.errors import ZipErrorList
+from mozharness.base.errors import TarErrorList, ZipErrorList
 from mozharness.base.log import SimpleFileLogger, MultiFileLogger, \
     LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL
 
+
 def platform_name():
     pm = PlatformMixin()
 
@@ -451,7 +452,7 @@ class ScriptMixin(PlatformMixin):
             **retry_args
         )
 
-    def download_unzip(self, url, parent_dir, target_unzip_dirs=None, halt_on_failure=True):
+    def download_unpack(self, url, parent_dir, extract_dirs=None, halt_on_failure=True):
         """Generic method to download and extract a zip file.
 
         The downloaded file will always be saved to the working directory and is not getting
@@ -461,29 +462,16 @@ class ScriptMixin(PlatformMixin):
             url (str): URL where the file to be downloaded is located.
             parent_dir (str): directory where the downloaded file will
                               be extracted to.
-            target_unzip_dirs (list, optional): directories inside the zip file to extract.
-                                                Defaults to `None`.
+            extract_dirs (list, optional): directories inside the archive file to extract.
+                                           Defaults to `None`.
             halt_on_failure (bool, optional): whether or not to redefine the
                                               log level as `FATAL` on errors. Defaults to True.
 
         """
         dirs = self.query_abs_dirs()
-        zipfile = self.download_file(url, parent_dir=dirs['abs_work_dir'],
+        archive = self.download_file(url, parent_dir=dirs['abs_work_dir'],
                                      error_level=FATAL)
-
-        command = self.query_exe('unzip', return_type='list')
-        # Always overwrite to not get an input in a hidden pipe if files already exist
-        command.extend(['-q', '-o', zipfile, '-d', parent_dir])
-        if target_unzip_dirs:
-            command.extend(target_unzip_dirs)
-        # TODO error_list: http://www.info-zip.org/mans/unzip.html#DIAGNOSTICS
-        # unzip return code 11 is 'no matching files were found'
-        self.run_command(command,
-                         error_list=ZipErrorList,
-                         halt_on_failure=halt_on_failure,
-                         fatal_exit_code=3,
-                         success_codes=[0, 11],
-                         )
+        self.unpack(archive, parent_dir, extract_dirs, halt_on_failure)
 
     def load_json_url(self, url, error_level=None, *args, **kwargs):
         """ Returns a json object from a url (it retries). """
@@ -1395,26 +1383,55 @@ class ScriptMixin(PlatformMixin):
                 self.log(msg, error_level=error_level)
         os.utime(file_name, times)
 
-    def unpack(self, filename, extract_to):
-        '''
+    def unpack(self, filename, extract_to,
+               extract_dirs=None, halt_on_failure=True):
+        """
         This method allows us to extract a file regardless of its extension
 
         Args:
             filename (str): filename of the compressed file.
             extract_to (str): where to extract the compressed file.
-        '''
-        # XXX: Make sure that filename has a extension of one of our supported file formats
+            extract_dirs (list, optional): directories inside the archive file to extract.
+                                           Defaults to `None`.
+            halt_on_failure (bool, optional): whether or not to redefine the
+                                              log level as `FATAL` on errors. Defaults to True.
+
+        """
         m = re.search('\.tar\.(bz2|gz)$', filename)
         if m:
+            # tar does not automatically create the target folder
+            if not os.path.exists(extract_to):
+                os.makedirs(extract_to)
+
             command = self.query_exe('tar', return_type='list')
-            tar_cmd = "jxfv"
-            if m.group(1) == "gz":
-                tar_cmd = "zxfv"
-            command.extend([tar_cmd, filename, "-C", extract_to])
-            self.run_command(command, halt_on_failure=True)
+            tar_args = 'zxfv' if m.group(1) == 'gz' else 'jxfv'
+            command.extend([tar_args, filename, '-C', extract_to])
+            if extract_dirs:
+                command.append('--wildcards')
+                command.extend(extract_dirs)
+            self.run_command(command,
+                             error_list=TarErrorList,
+                             fatal_exit_code=3,
+                             halt_on_failure=halt_on_failure,
+                             )
+
+        elif filename.endswith('.zip'):
+            command = self.query_exe('unzip', return_type='list')
+            # Always overwrite to not get an input in a hidden pipe if files already exist
+            command.extend(['-q', '-o', filename, '-d', extract_to])
+            if extract_dirs:
+                command.extend(extract_dirs)
+            # TODO error_list: http://www.info-zip.org/mans/unzip.html#DIAGNOSTICS
+            # unzip return code 11 is 'no matching files were found'
+            self.run_command(command,
+                             error_list=ZipErrorList,
+                             fatal_exit_code=3,
+                             halt_on_failure=halt_on_failure,
+                             success_codes=[0, 11],
+                             )
+
         else:
-            # XXX implement
-            pass
+            raise NotImplementedError('No extraction method found for: %s' % filename)
 
 
 def PreScriptRun(func):

testing/mozharness/mozharness/mozilla/testing/talos.py

@@ -274,13 +274,13 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
         if self.query_pagesets_url():
             self.info("Downloading pageset...")
             src_talos_pageset = os.path.join(src_talos_webdir, 'tests')
-            self.download_unzip(self.pagesets_url, src_talos_pageset)
+            self.download_unpack(self.pagesets_url, src_talos_pageset)
 
     # Action methods. {{{1
     # clobber defined in BaseScript
     # read_buildbot_config defined in BuildbotMixin
 
-    def download_and_extract(self, target_unzip_dirs=None, suite_categories=None):
+    def download_and_extract(self, extract_dirs=None, suite_categories=None):
         return super(Talos, self).download_and_extract(
             suite_categories=['common', 'talos']
         )

testing/mozharness/mozharness/mozilla/testing/testbase.py

@@ -396,7 +396,7 @@ You can set this by:
                   pprint.pformat(package_requirements))
         return package_requirements
 
-    def _download_test_packages(self, suite_categories, target_unzip_dirs):
+    def _download_test_packages(self, suite_categories, extract_dirs):
         # Some platforms define more suite categories/names than others.
         # This is a difference in the convention of the configs more than
         # to how these tests are run, so we pave over these differences here.
@@ -430,21 +430,21 @@ You can set this by:
                       (target_packages, category))
             for file_name in target_packages:
                 target_dir = test_install_dir
-                unzip_dirs = target_unzip_dirs
+                unpack_dirs = extract_dirs
                 if "jsshell-" in file_name or file_name == "target.jsshell.zip":
                     self.info("Special-casing the jsshell zip file")
-                    unzip_dirs = None
+                    unpack_dirs = None
                     target_dir = dirs['abs_test_bin_dir']
                 url = self.query_build_dir_url(file_name)
-                self.download_unzip(url, target_dir,
-                                     target_unzip_dirs=unzip_dirs)
+                self.download_unpack(url, target_dir,
+                                     extract_dirs=unpack_dirs)
 
-    def _download_test_zip(self, target_unzip_dirs=None):
+    def _download_test_zip(self, extract_dirs=None):
         dirs = self.query_abs_dirs()
         test_install_dir = dirs.get('abs_test_install_dir',
                                     os.path.join(dirs['abs_work_dir'], 'tests'))
-        self.download_unzip(self.test_url, test_install_dir,
-                             target_unzip_dirs=target_unzip_dirs)
+        self.download_unpack(self.test_url, test_install_dir,
+                             extract_dirs=extract_dirs)
 
     def structured_output(self, suite_category):
         """Defines whether structured logging is in use in this configuration. This
@@ -491,9 +491,9 @@ You can set this by:
 
         self.set_buildbot_property("symbols_url", self.symbols_url,
                                    write_to_file=True)
-        self.download_unzip(self.symbols_url, self.symbols_path)
+        self.download_unpack(self.symbols_url, self.symbols_path)
 
-    def download_and_extract(self, target_unzip_dirs=None, suite_categories=None):
+    def download_and_extract(self, extract_dirs=None, suite_categories=None):
         """
         download and extract test zip / download installer
         """
@@ -516,7 +516,7 @@ You can set this by:
                            ' package data at "%s" will be ignored.' %
                            (self.config.get('test_url'), self.test_packages_url))
 
-            self._download_test_zip(target_unzip_dirs)
+            self._download_test_zip(extract_dirs)
         else:
             if not self.test_packages_url:
                 # The caller intends to download harness specific packages, but doesn't know
@@ -526,7 +526,7 @@ You can set this by:
                 self.test_packages_url = self.query_build_dir_url('test_packages.json')
 
             suite_categories = suite_categories or ['common']
-            self._download_test_packages(suite_categories, target_unzip_dirs)
+            self._download_test_packages(suite_categories, extract_dirs)
 
         self._download_installer()
         if self.config.get('download_symbols'):

testing/mozharness/mozharness/mozilla/testing/unittest.py

@@ -221,7 +221,7 @@ class EmulatorMixin(object):
         dirs = self.query_abs_dirs()
         self.mkdir_p(dirs['abs_emulator_dir'])
         if self.config.get('emulator_url'):
-            self.download_unzip(self.config['emulator_url'], dirs['abs_emulator_dir'])
+            self.download_unpack(self.config['emulator_url'], dirs['abs_emulator_dir'])
         elif self.config.get('emulator_manifest'):
             manifest_path = self.create_tooltool_manifest(self.config['emulator_manifest'])
             do_unzip = True

testing/mozharness/scripts/android_emulator_unittest.py

@@ -627,7 +627,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
             robocop_url = self.installer_url[:self.installer_url.rfind('/')] + '/robocop.apk'
             self.info("Downloading robocop...")
             self.download_file(robocop_url, 'robocop.apk', dirs['abs_work_dir'], error_level=FATAL)
-        self.download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
+        self.download_unpack(self.host_utils_url, dirs['abs_xre_dir'])
 
     def install(self):
         """

testing/mozharness/scripts/android_panda.py

@@ -383,7 +383,7 @@ class PandaTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixin, Bu
         dirs = self.query_abs_dirs()
         self.host_utils_url = c['hostutils_url']
         # get the zip and extract it
-        self.download_unzip(self.host_utils_url, dirs['abs_hostutils_dir'])
+        self.download_unpack(self.host_utils_url, dirs['abs_hostutils_dir'])
 
     def _install_app(self):
         c = self.config

testing/mozharness/scripts/android_panda_talos.py

@@ -299,7 +299,7 @@ class PandaTalosTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixi
         self.rmtree(dirs['abs_talosdata_dir'])
         self.mkdir_p(dirs['abs_talosdata_dir'])
         self.mkdir_p(dirs['abs_symbols_dir'])
-        self.download_unzip(self.installer_url,
+        self.download_unpack(self.installer_url,
                              dirs['abs_fennec_dir'])
         #this is ugly but you can't specify a file in download_unzip to extract the file to, by default it's the abs_work_dir
         #should think of a better way
@@ -317,10 +317,10 @@ class PandaTalosTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixi
                            error_level=FATAL)
         self.symbols_url = self.query_symbols_url()
 
-        self.download_unzip(self.symbols_url,
+        self.download_unpack(self.symbols_url,
                              dirs['abs_symbols_dir'])
 
-        self.download_unzip(self.config['retry_url'],
+        self.download_unpack(self.config['retry_url'],
                              dirs['abs_talosdata_dir'])
 
         taloscode = self.config.get("talos_from_code_url")

testing/mozharness/scripts/androidx86_emulator_unittest.py

@@ -681,7 +681,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
                 self._download_robocop_apk()
                 break
 
-        self.download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
+        self.download_unpack(self.host_utils_url, dirs['abs_xre_dir'])
 
     def install(self):
         assert self.installer_path is not None, \

testing/mozharness/scripts/b2g_emulator_unittest.py

@@ -208,7 +208,7 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
                          error_list=TarErrorList,
                          halt_on_failure=True, fatal_exit_code=3)
 
-        self.download_unzip(self.config['xre_url'],
+        self.download_unpack(self.config['xre_url'],
                              dirs['abs_xre_dir'])
 
         if self.config.get('busybox_url'):

testing/mozharness/scripts/desktop_unittest.py

@@ -475,20 +475,20 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
         """
         c = self.config
 
-        target_unzip_dirs = None
+        extract_dirs = None
         if c['specific_tests_zip_dirs']:
-            target_unzip_dirs = list(c['minimum_tests_zip_dirs'])
+            extract_dirs = list(c['minimum_tests_zip_dirs'])
             for category in c['specific_tests_zip_dirs'].keys():
                 if c['run_all_suites'] or self._query_specified_suites(category) \
                         or 'run-tests' not in self.actions:
-                    target_unzip_dirs.extend(c['specific_tests_zip_dirs'][category])
+                    extract_dirs.extend(c['specific_tests_zip_dirs'][category])
 
         if c.get('run_all_suites'):
             target_categories = SUITE_CATEGORIES
         else:
             target_categories = [cat for cat in SUITE_CATEGORIES
                                  if self._query_specified_suites(cat) is not None]
-        super(DesktopUnittest, self).download_and_extract(target_unzip_dirs=target_unzip_dirs,
+        super(DesktopUnittest, self).download_and_extract(extract_dirs=extract_dirs,
                                                           suite_categories=target_categories)
 
     # pull defined in VCSScript.

testing/mozharness/scripts/web_platform_tests.py

@@ -165,12 +165,12 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin):
 
     def download_and_extract(self):
         super(WebPlatformTest, self).download_and_extract(
-            target_unzip_dirs=["bin/*",
-                               "config/*",
-                               "mozbase/*",
-                               "marionette/*",
-                               "tools/wptserve/*",
-                               "web-platform/*"],
+            extract_dirs=["bin/*",
+                          "config/*",
+                          "mozbase/*",
+                          "marionette/*",
+                          "tools/wptserve/*",
+                          "web-platform/*"],
             suite_categories=["web-platform"])
 
     def run_tests(self):