Fix and refactor recognition of container types. (#22857)

* Fix and refactor recognition of container types.

* Use retry instead of redundant code when ejecting DMGs.

* Use BOM file for extracting DMGs.

* Add `lzma` compression.

ci/travis/install.sh

@@ -22,5 +22,6 @@ if must_run_tests; then
   # install Formulae and Casks without which some tests would be skipped
   brew_install cabextract
   brew_install unar
+  brew_install xz
   run brew cask install Casks/adobe-air.rb
 fi

lib/hbc/container.rb

@@ -8,6 +8,7 @@ require "hbc/container/criteria"
 require "hbc/container/dmg"
 require "hbc/container/generic_unar"
 require "hbc/container/gzip"
+require "hbc/container/lzma"
 require "hbc/container/naked"
 require "hbc/container/otf"
 require "hbc/container/pkg"
@@ -17,6 +18,7 @@ require "hbc/container/tar"
 require "hbc/container/ttf"
 require "hbc/container/rar"
 require "hbc/container/xar"
+require "hbc/container/xz"
 require "hbc/container/zip"
 
 class Hbc::Container
@@ -30,11 +32,13 @@ class Hbc::Container
       Hbc::Container::Dmg,
       Hbc::Container::SevenZip,
       Hbc::Container::Sit,
-      Hbc::Container::Tar, # or compressed tar
       Hbc::Container::Rar,
       Hbc::Container::Zip,
-      Hbc::Container::Bzip2,
-      Hbc::Container::Gzip, # pure gzip, not tar/gzip
+      Hbc::Container::Tar,   # or compressed tar (bzip2/gzip/lzma/xz)
+      Hbc::Container::Bzip2, # pure bzip2
+      Hbc::Container::Gzip,  # pure gzip
+      Hbc::Container::Lzma,  # pure lzma
+      Hbc::Container::Xz,    # pure xz
       Hbc::Container::Xar,
     ]
     # for explicit use only (never autodetected):

lib/hbc/container/bzip2.rb

@@ -2,7 +2,7 @@ require "tmpdir"
 
 class Hbc::Container::Bzip2 < Hbc::Container::Base
   def self.me?(criteria)
-    criteria.file.include? "compressed-encoding=application/x-bzip2;"
+    criteria.magic_number(%r{^BZh}n)
   end
 
   def extract

lib/hbc/container/cab.rb

@@ -2,17 +2,20 @@ require "tmpdir"
 
 class Hbc::Container::Cab < Hbc::Container::Base
   def self.me?(criteria)
-    (criteria.file.include?("application/octet-stream;") ||
-     criteria.file.include?("application/vnd.ms-cab-compressed;")) &&
-      !criteria.cabextract.nil? &&
-      criteria.cabextract.include?("All done, no errors")
+    cabextract = Hbc.homebrew_prefix.join("bin", "cabextract")
+
+    criteria.magic_number(%r{^MSCF}n) &&
+      cabextract.exist? &&
+      criteria.command.run(cabextract, args: ["-t", "--", criteria.path.to_s]).stderr.empty?
   end
 
   def extract
-    cabextract = Hbc.homebrew_prefix.join("bin/cabextract")
-    unless Pathname.new(cabextract).exist?
-      raise Hbc::CaskError, "Expected to find cabextract executable. Cask '#{@cask}' must add: depends_on :formula => 'cabextract'"
+    cabextract = Hbc.homebrew_prefix.join("bin", "cabextract")
+
+    unless cabextract.exist?
+      raise Hbc::CaskError, "Expected to find cabextract executable. Cask '#{@cask}' must add: depends_on formula: 'cabextract'"
     end
+
     Dir.mktmpdir do |unpack_dir|
       @command.run!(cabextract, args: ["-d", unpack_dir, "--", @path])
       @command.run!("/usr/bin/ditto", args: ["--", unpack_dir, @cask.staged_path])

lib/hbc/container/criteria.rb

@@ -1,46 +1,18 @@
 class Hbc::Container::Criteria
-  attr_reader :path
+  attr_reader :path, :command
 
   def initialize(path, command)
     @path = path
     @command = command
   end
 
-  def file
-    @file ||= @command.run("/usr/bin/file", args: ["-Izb", "--", path]).stdout
+  def extension(regex)
+    path.extname.sub(%r{^\.}, "") =~ Regexp.new(regex.source, regex.options | Regexp::IGNORECASE)
   end
 
-  def imageinfo
-    @imageinfo ||= @command.run("/usr/bin/hdiutil",
-                                # realpath is a failsafe against unusual filenames
-                                args:         ["imageinfo", Pathname.new(path).realpath],
-                                print_stderr: false).stdout
-  end
-
-  def cabextract
-    if Hbc.homebrew_prefix.join("bin/cabextract").exist?
-      @cabextract ||= @command.run(Hbc.homebrew_prefix.join("bin/cabextract"),
-                                   args:         ["-t", "--", path],
-                                   print_stderr: false).stdout
-    end
-  end
-
-  def lsar
-    if Hbc.homebrew_prefix.join("bin/lsar").exist?
-      @lsar ||= @command.run(Hbc.homebrew_prefix.join("bin/lsar"),
-                             args:         ["-l", "-t", "--", path],
-                             print_stderr: false).stdout
-    end
-  end
-
-  def extension(test)
-    path.extname.sub(%r{\A\.}, "").casecmp(test).zero?
-  end
-
-  def magic_number(num, test)
-    File.open(path, "rb") do |file|
-      bytes = file.read(num).unpack("C*")
-      bytes == test
-    end
+  def magic_number(regex)
+    # 262: length of the longest regex (currently: Hbc::Container::Tar)
+    @magic_number ||= File.open(path, "rb") { |f| f.read(262) }
+    @magic_number =~ regex
   end
 end

lib/hbc/container/dmg.rb

@@ -1,6 +1,11 @@
+require "tempfile"
+
 class Hbc::Container::Dmg < Hbc::Container::Base
   def self.me?(criteria)
-    !criteria.imageinfo.empty?
+    !criteria.command.run("/usr/bin/hdiutil",
+                          # realpath is a failsafe against unusual filenames
+                          args:         ["imageinfo", Pathname.new(criteria.path).realpath],
+                          print_stderr: false).stdout.empty?
   end
 
   attr_reader :mounts
@@ -13,17 +18,17 @@ class Hbc::Container::Dmg < Hbc::Container::Base
     mount!
     assert_mounts_found
     @mounts.each do |mount|
-      @command.run("/usr/bin/ditto",
-                   # TODO: per https://github.com/caskroom/homebrew-cask/issues/6382, ditto
-                   #       complains to stderr about unreadable .Trashes directories, so all
-                   #       stderr output is silenced for now.  But better solutions would be
-                   #       - use the --bom option to ditto to selectively avoid certain files
-                   #         - .Trashes
-                   #         - symlinks to Applications
-                   #       - or support some type of text filter to be passed to
-                   #         :print_stderr instead of true/false
-                   print_stderr: false,
-                   args:         ["--", mount, @cask.staged_path])
+      Tempfile.open(["", ".bom"]) do |bomfile|
+        bomfile.close
+
+        Tempfile.open(["", ".list"]) do |filelist|
+          filelist.write(bom_filelist_from_path(mount))
+          filelist.close
+
+          @command.run("/usr/bin/mkbom", args: ["-s", "-i", filelist.path, "--", bomfile.path])
+          @command.run("/usr/bin/ditto", args: ["--bom", bomfile.path, "--", mount, @cask.staged_path])
+        end
+      end
     end
   ensure
     eject!
@@ -38,6 +43,61 @@ class Hbc::Container::Dmg < Hbc::Container::Base
     @mounts = mounts_from_plist(plist)
   end
 
+  def eject!
+    @mounts.each do |mount|
+      # realpath is a failsafe against unusual filenames
+      mountpath = Pathname.new(mount).realpath
+      next unless mountpath.exist?
+
+      begin
+        tries ||= 2
+        @command.run("/usr/sbin/diskutil",
+                     args:         ["eject", mountpath],
+                     print_stderr: false)
+
+        raise Hbc::CaskError, "Failed to eject #{mountpath}" if mountpath.exist?
+      rescue Hbc::CaskError => e
+        raise e if (tries -= 1).zero?
+        sleep 1
+        retry
+      end
+    end
+  end
+
+  private
+
+  def bom_filelist_from_path(mount)
+    mountpath = Pathname.new(mount).realpath
+
+    paths = Dir.glob(mountpath.join("**", "*"), File::FNM_DOTMATCH)
+               .map { |path| Pathname.new(path).relative_path_from(mountpath) }
+
+    paths = paths.reject { |path|
+      path = mountpath.join(path.sub(%r{/.*}, ""))
+
+      # unnecessary DMG metadata
+      %w[
+        .background
+        .com.apple.timemachine.donotpresent
+        .DocumentRevisions-V100
+        .DS_Store
+        .fseventsd
+        .Spotlight-V100
+        .TemporaryItems
+        .Trashes
+        .VolumeIcon.icns
+      ].include?(path.basename.to_s) ||
+
+        # symlinks to system directories (commonly to /Applications)
+        (path.symlink? &&
+         Hbc::MacOS::SYSTEM_DIRS.include?(Pathname.new(File.readlink(path))))
+    }
+
+    paths.map(&:to_s)
+         .map { |path| path.prepend(path == "." ? "" : "./").concat("\n") }
+         .join
+  end
+
   def mounts_from_plist(plist)
     return [] unless plist.respond_to?(:fetch)
     plist.fetch("system-entities", []).map { |entity|
@@ -48,22 +108,4 @@ class Hbc::Container::Dmg < Hbc::Container::Base
   def assert_mounts_found
     raise Hbc::CaskError, "No mounts found in '#{@path}'; perhaps it is a bad DMG?" if @mounts.empty?
   end
-
-  def eject!
-    @mounts.each do |mount|
-      # realpath is a failsafe against unusual filenames
-      mountpath = Pathname.new(mount).realpath
-      next unless mountpath.exist?
-      @command.run("/usr/sbin/diskutil",
-                   args:         ["eject", mountpath],
-                   print_stderr: false)
-      next unless mountpath.exist?
-      sleep 1
-      @command.run("/usr/sbin/diskutil",
-                   args:         ["eject", mountpath],
-                   print_stderr: false)
-      next unless mountpath.exist?
-      raise Hbc::CaskError, "Failed to eject #{mountpath}"
-    end
-  end
 end

lib/hbc/container/generic_unar.rb

@@ -2,14 +2,20 @@ require "tmpdir"
 
 class Hbc::Container::GenericUnar < Hbc::Container::Base
   def self.me?(criteria)
-    !criteria.lsar.nil? && criteria.lsar.include?("passed, 0 failed")
+    lsar = Hbc.homebrew_prefix.join("bin", "lsar")
+    lsar.exist? &&
+      criteria.command.run(lsar,
+                           args:         ["-l", "-t", "--", criteria.path],
+                           print_stderr: false).stdout.chomp.end_with?("passed, 0 failed.")
   end
 
   def extract
-    unar = Hbc.homebrew_prefix.join("bin/unar")
-    unless Pathname.new(unar).exist?
-      raise Hbc::CaskError, "Expected to find unar executable. Cask #{@cask} must add: depends_on :formula => 'unar'"
+    unar = Hbc.homebrew_prefix.join("bin", "unar")
+
+    unless unar.exist?
+      raise Hbc::CaskError, "Expected to find unar executable. Cask #{@cask} must add: depends_on formula: 'unar'"
     end
+
     Dir.mktmpdir do |unpack_dir|
       @command.run!(unar, args: ["-force-overwrite", "-quiet", "-no-directory", "-output-directory", unpack_dir, "--", @path])
       @command.run!("/usr/bin/ditto", args: ["--", unpack_dir, @cask.staged_path])

lib/hbc/container/gzip.rb

@@ -1,11 +1,8 @@
-
-# for pure gzip only, not tar-gzip (.tgz or .tar.gz)
-
 require "tmpdir"
 
 class Hbc::Container::Gzip < Hbc::Container::Base
   def self.me?(criteria)
-    criteria.file.include? "compressed-encoding=application/x-gzip;"
+    criteria.magic_number(%r{^\037\213}n)
   end
 
   def extract

lib/hbc/container/lzma.rb

@@ -0,0 +1,21 @@
+require "tmpdir"
+
+class Hbc::Container::Lzma < Hbc::Container::Base
+  def self.me?(criteria)
+    criteria.magic_number(%r{^\]\000\000\200\000}n)
+  end
+
+  def extract
+    unlzma = Hbc.homebrew_prefix.join("bin", "unlzma")
+
+    unless unlzma.exist?
+      raise Hbc::CaskError, "Expected to find unlzma executable. Cask '#{@cask}' must add: depends_on formula: 'lzma'"
+    end
+
+    Dir.mktmpdir do |unpack_dir|
+      @command.run!("/usr/bin/ditto", args: ["--", @path, unpack_dir])
+      @command.run!(unlzma, args: ["-q", "--", Pathname(unpack_dir).join(@path.basename)])
+      @command.run!("/usr/bin/ditto", args: ["--", unpack_dir, @cask.staged_path])
+    end
+  end
+end

lib/hbc/container/otf.rb

@@ -1,7 +1,5 @@
 class Hbc::Container::Otf < Hbc::Container::Naked
   def self.me?(criteria)
-    criteria.extension("otf") &&
-      (criteria.file.include?("application/vnd.ms-opentype") ||
-       criteria.file.include?("application/x-font-ttf"))
+    criteria.magic_number(%r{^OTTO}n)
   end
 end

lib/hbc/container/pkg.rb

@@ -1,8 +1,7 @@
 class Hbc::Container::Pkg < Hbc::Container::Naked
   def self.me?(criteria)
-    (criteria.extension("pkg") ||
-     criteria.extension("mpkg")) &&
-      (criteria.file.include?("application/x-directory") ||
-       criteria.magic_number(4, "xar!".unpack("C*")))
+    criteria.extension(%r{m?pkg$}) &&
+      (criteria.path.directory? ||
+       criteria.magic_number(%r{^xar!}n))
   end
 end

lib/hbc/container/rar.rb

@@ -1,9 +1,6 @@
 class Hbc::Container::Rar < Hbc::Container::GenericUnar
   def self.me?(criteria)
-    (criteria.file.include?("application/x-rar;") ||
-     criteria.file.include?("application/octet-stream;")) &&
-      !criteria.lsar.nil? &&
-      criteria.lsar.split("\n").first.split(":").last.include?("RAR") &&
+    criteria.magic_number(%r{^Rar!}n) &&
       super
   end
 end

lib/hbc/container/seven_zip.rb

@@ -1,11 +1,7 @@
 class Hbc::Container::SevenZip < Hbc::Container::GenericUnar
   def self.me?(criteria)
     # TODO: cover self-extracting archives
-    criteria.extension("7z") &&
-      criteria.file.include?("application/octet-stream;") &&
-      criteria.magic_number(2, "7z".unpack("C*")) &&
-      !criteria.lsar.nil? &&
-      criteria.lsar.split("\n").first.split(":").last.include?("7-Zip") &&
+    criteria.magic_number(%r{^7z}n) &&
       super
   end
 end

lib/hbc/container/sit.rb

@@ -1,8 +1,6 @@
 class Hbc::Container::Sit < Hbc::Container::GenericUnar
   def self.me?(criteria)
-    criteria.file.include?("application/x-stuffit") &&
-      !criteria.lsar.nil? &&
-      criteria.lsar.split("\n").first.split(":").last.include?("StuffIt") &&
+    criteria.magic_number(%r{^StuffIt}n) &&
       super
   end
 end

lib/hbc/container/tar.rb

@@ -2,12 +2,14 @@ require "tmpdir"
 
 class Hbc::Container::Tar < Hbc::Container::Base
   def self.me?(criteria)
-    criteria.file.include? "application/x-tar"
+    criteria.magic_number(%r{^.{257}ustar}n) ||
+      # or compressed tar (bzip2/gzip/lzma/xz)
+      IO.popen(["/usr/bin/tar", "-t", "-f", criteria.path.to_s], err: "/dev/null") { |io| !io.read(1).nil? }
   end
 
   def extract
     Dir.mktmpdir do |unpack_dir|
-      @command.run!("/usr/bin/tar", args: ["xf", @path, "-C", unpack_dir])
+      @command.run!("/usr/bin/tar", args: ["-x", "-f", @path, "-C", unpack_dir])
       @command.run!("/usr/bin/ditto", args: ["--", unpack_dir, @cask.staged_path])
     end
   end

lib/hbc/container/ttf.rb

@@ -1,9 +1,8 @@
 class Hbc::Container::Ttf < Hbc::Container::Naked
   def self.me?(criteria)
-    (criteria.extension("ttf") &&
-     (criteria.file.include?("application/x-font-ttf") ||
-      criteria.magic_number(4, "true".unpack("C*")))) ||
-      (criteria.extension("ttc") &&
-       criteria.magic_number(4, "ttcf".unpack("C*")))
+    # TrueType Font
+    criteria.magic_number(%r{^\000\001\000\000\000}n) ||
+      # Truetype Font Collection
+      criteria.magic_number(%r{^ttcf}n)
   end
 end

lib/hbc/container/xar.rb

@@ -2,12 +2,12 @@ require "tmpdir"
 
 class Hbc::Container::Xar < Hbc::Container::Base
   def self.me?(criteria)
-    criteria.magic_number(4, "xar!".unpack("C*"))
+    criteria.magic_number(%r{^xar!}n)
   end
 
   def extract
     Dir.mktmpdir do |unpack_dir|
-      @command.run!("/usr/bin/xar", args: ["-xf", @path, "-C", unpack_dir])
+      @command.run!("/usr/bin/xar", args: ["-x", "-f", @path, "-C", unpack_dir])
       @command.run!("/usr/bin/ditto", args: ["--", unpack_dir, @cask.staged_path])
     end
   end

lib/hbc/container/xz.rb

@@ -0,0 +1,21 @@
+require "tmpdir"
+
+class Hbc::Container::Xz < Hbc::Container::Base
+  def self.me?(criteria)
+    criteria.magic_number(%r{^\xFD7zXZ\x00}n)
+  end
+
+  def extract
+    unxz = Hbc.homebrew_prefix.join("bin", "unxz")
+
+    unless unxz.exist?
+      raise Hbc::CaskError, "Expected to find unxz executable. Cask '#{@cask}' must add: depends_on formula: 'xz'"
+    end
+
+    Dir.mktmpdir do |unpack_dir|
+      @command.run!("/usr/bin/ditto", args: ["--", @path, unpack_dir])
+      @command.run!(unxz, args: ["-q", "--", Pathname(unpack_dir).join(@path.basename)])
+      @command.run!("/usr/bin/ditto", args: ["--", unpack_dir, @cask.staged_path])
+    end
+  end
+end

lib/hbc/container/zip.rb

@@ -1,9 +1,9 @@
 class Hbc::Container::Zip < Hbc::Container::Base
   def self.me?(criteria)
-    criteria.file.include? "compressed-encoding=application/zip;"
+    criteria.magic_number(%r{^PK(\003\004|\005\006)}n)
   end
 
   def extract
-    @command.run!("/usr/bin/ditto", args: ["-xk", "--", @path, @cask.staged_path])
+    @command.run!("/usr/bin/ditto", args: ["-x", "-k", "--", @path, @cask.staged_path])
   end
 end

test/cask/installer_test.rb

@@ -158,6 +158,38 @@ describe Hbc::Installer do
       file.must_be :file?
     end
 
+    it "works with xz-based Casks" do
+      skip("unxz not installed") unless Hbc.homebrew_prefix.join("bin", "unxz").exist?
+      asset = Hbc.load("xzipped-asset")
+      empty = stub(formula: [], cask: [], macos: nil, arch: nil, x11: nil)
+      asset.stubs(:depends_on).returns(empty)
+
+      shutup do
+        Hbc::Installer.new(asset).install
+      end
+
+      dest_path = Hbc.caskroom.join("xzipped-asset", asset.version)
+      dest_path.must_be :directory?
+      file = Hbc.appdir.join("xzipped-asset-#{asset.version}")
+      file.must_be :file?
+    end
+
+    it "works with lzma-based Casks" do
+      skip("unlzma not installed") unless Hbc.homebrew_prefix.join("bin", "unlzma").exist?
+      asset = Hbc.load("lzma-asset")
+      empty = stub(formula: [], cask: [], macos: nil, arch: nil, x11: nil)
+      asset.stubs(:depends_on).returns(empty)
+
+      shutup do
+        Hbc::Installer.new(asset).install
+      end
+
+      dest_path = Hbc.caskroom.join("lzma-asset", asset.version)
+      dest_path.must_be :directory?
+      file = Hbc.appdir.join("lzma-asset-#{asset.version}")
+      file.must_be :file?
+    end
+
     it "blows up on a bad checksum" do
       bad_checksum = Hbc.load("bad-checksum")
       lambda {