Merge pull request #8538 from JosJuice/wia

Add support for the WIA and RVZ disc image formats

Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java

@@ -22,7 +22,7 @@ import java.util.List;
 public final class FileBrowserHelper
 {
   public static final HashSet<String> GAME_EXTENSIONS = new HashSet<>(Arrays.asList(
-          "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wad", "dol", "elf", "dff"));
+          "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wia", "rvz", "wad", "dol", "elf", "dff"));
 
   public static final HashSet<String> RAW_EXTENSION = new HashSet<>(Collections.singletonList(
           "raw"));

Source/Core/Core/Boot/Boot.cpp

@@ -159,7 +159,7 @@ BootParameters::GenerateFromFile(std::vector<std::string> paths,
     paths.clear();
 
   static const std::unordered_set<std::string> disc_image_extensions = {
-      {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".dol", ".elf"}};
+      {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".wia", ".rvz", ".dol", ".elf"}};
   if (disc_image_extensions.find(extension) != disc_image_extensions.end() || is_drive)
   {
     std::unique_ptr<DiscIO::VolumeDisc> disc = DiscIO::CreateDisc(path);

Source/Core/DiscIO/Blob.cpp

@@ -20,6 +20,7 @@
 #include "DiscIO/DriveBlob.h"
 #include "DiscIO/FileBlob.h"
 #include "DiscIO/TGCBlob.h"
+#include "DiscIO/WIABlob.h"
 #include "DiscIO/WbfsBlob.h"
 
 namespace DiscIO
@@ -205,6 +206,10 @@ std::unique_ptr<BlobReader> CreateBlobReader(const std::string& filename)
     return TGCFileReader::Create(std::move(file));
   case WBFS_MAGIC:
     return WbfsFileReader::Create(std::move(file), filename);
+  case WIA_MAGIC:
+    return WIAFileReader::Create(std::move(file), filename);
+  case RVZ_MAGIC:
+    return RVZFileReader::Create(std::move(file), filename);
   default:
     if (auto directory_blob = DirectoryBlobReader::Create(filename))
       return std::move(directory_blob);

Source/Core/DiscIO/Blob.h

@@ -25,6 +25,8 @@
 
 namespace DiscIO
 {
+enum class WIARVZCompressionType : u32;
+
 // Increment CACHE_REVISION (GameFileCache.cpp) if the enum below is modified
 enum class BlobType
 {
@@ -34,7 +36,9 @@ enum class BlobType
   GCZ,
   CISO,
   WBFS,
-  TGC
+  TGC,
+  WIA,
+  RVZ,
 };
 
 class BlobReader
@@ -172,5 +176,9 @@ bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path,
 bool ConvertToPlain(BlobReader* infile, const std::string& infile_path,
                     const std::string& outfile_path, CompressCB callback = nullptr,
                     void* arg = nullptr);
+bool ConvertToWIAOrRVZ(BlobReader* infile, const std::string& infile_path,
+                       const std::string& outfile_path, bool rvz,
+                       WIARVZCompressionType compression_type, int compression_level,
+                       int chunk_size, CompressCB callback = nullptr, void* arg = nullptr);
 
 }  // namespace DiscIO

Source/Core/DiscIO/CMakeLists.txt

@@ -21,6 +21,8 @@ add_library(discio
   FileSystemGCWii.h
   Filesystem.cpp
   Filesystem.h
+  LaggedFibonacciGenerator.cpp
+  LaggedFibonacciGenerator.h
   MultithreadedCompressor.h
   NANDImporter.cpp
   NANDImporter.h
@@ -42,6 +44,10 @@ add_library(discio
   VolumeWii.h
   WbfsBlob.cpp
   WbfsBlob.h
+  WIABlob.cpp
+  WIABlob.h
+  WIACompression.cpp
+  WIACompression.h
   WiiEncryptionCache.cpp
   WiiEncryptionCache.h
   WiiSaveBanner.cpp
@@ -49,6 +55,11 @@ add_library(discio
 )
 
 target_link_libraries(discio
+PUBLIC
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  zstd
+
 PRIVATE
   minizip
   pugixml

Source/Core/DiscIO/DiscIO.vcxproj

@@ -55,6 +55,7 @@
     <ClCompile Include="FileBlob.cpp" />
     <ClCompile Include="Filesystem.cpp" />
     <ClCompile Include="FileSystemGCWii.cpp" />
+    <ClCompile Include="LaggedFibonacciGenerator.cpp" />
     <ClCompile Include="NANDImporter.cpp" />
     <ClCompile Include="ScrubbedBlob.cpp" />
     <ClCompile Include="TGCBlob.cpp" />
@@ -65,6 +66,8 @@
     <ClCompile Include="VolumeWad.cpp" />
     <ClCompile Include="VolumeWii.cpp" />
     <ClCompile Include="WbfsBlob.cpp" />
+    <ClCompile Include="WIABlob.cpp" />
+    <ClCompile Include="WIACompression.cpp" />
     <ClCompile Include="WiiEncryptionCache.cpp" />
     <ClCompile Include="WiiSaveBanner.cpp" />
   </ItemGroup>
@@ -80,6 +83,7 @@
     <ClInclude Include="FileBlob.h" />
     <ClInclude Include="Filesystem.h" />
     <ClInclude Include="FileSystemGCWii.h" />
+    <ClInclude Include="LaggedFibonacciGenerator.h" />
     <ClInclude Include="MultithreadedCompressor.h" />
     <ClInclude Include="NANDImporter.h" />
     <ClInclude Include="ScrubbedBlob.h" />
@@ -91,6 +95,8 @@
     <ClInclude Include="VolumeWad.h" />
     <ClInclude Include="VolumeWii.h" />
     <ClInclude Include="WbfsBlob.h" />
+    <ClInclude Include="WIABlob.h" />
+    <ClInclude Include="WIACompression.h" />
     <ClInclude Include="WiiEncryptionCache.h" />
     <ClInclude Include="WiiSaveBanner.h" />
   </ItemGroup>
@@ -110,6 +116,15 @@
     <ProjectReference Include="$(ExternalsDir)pugixml\pugixml.vcxproj">
       <Project>{38fee76f-f347-484b-949c-b4649381cffb}</Project>
     </ProjectReference>
+    <ProjectReference Include="$(ExternalsDir)bzip2\bzip2.vcxproj">
+      <Project>{055a775f-b4f5-4970-9240-f6cf7661f37b}</Project>
+    </ProjectReference>
+    <ProjectReference Include="$(ExternalsDir)liblzma\liblzma.vcxproj">
+      <Project>{1d8c51d2-ffa4-418e-b183-9f42b6a6717e}</Project>
+    </ProjectReference>
+    <ProjectReference Include="$(ExternalsDir)zstd\zstd.vcxproj">
+      <Project>{1bea10f3-80ce-4bc4-9331-5769372cdf99}</Project>
+    </ProjectReference>
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

Source/Core/DiscIO/DiscIO.vcxproj.filters

@@ -90,6 +90,15 @@
     <ClCompile Include="ScrubbedBlob.cpp">
       <Filter>Volume\Blob</Filter>
     </ClCompile>
+    <ClCompile Include="WIABlob.cpp">
+      <Filter>Volume\Blob</Filter>
+    </ClCompile>
+    <ClCompile Include="LaggedFibonacciGenerator.cpp">
+      <Filter>Volume\Blob</Filter>
+    </ClCompile>
+    <ClCompile Include="WIACompression.cpp">
+      <Filter>Volume\Blob</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="DiscScrubber.h">
@@ -164,6 +173,15 @@
     <ClInclude Include="MultithreadedCompressor.h">
       <Filter>Volume\Blob</Filter>
     </ClInclude>
+    <ClInclude Include="WIABlob.h">
+      <Filter>Volume\Blob</Filter>
+    </ClInclude>
+    <ClInclude Include="LaggedFibonacciGenerator.h">
+      <Filter>Volume\Blob</Filter>
+    </ClInclude>
+    <ClInclude Include="WIACompression.h">
+      <Filter>Volume\Blob</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Text Include="CMakeLists.txt" />

Source/Core/DiscIO/LaggedFibonacciGenerator.cpp

@@ -0,0 +1,212 @@
+// This file is under the public domain.
+
+#include "DiscIO/LaggedFibonacciGenerator.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstring>
+
+#include "Common/Align.h"
+#include "Common/Assert.h"
+#include "Common/CommonTypes.h"
+#include "Common/Swap.h"
+
+namespace DiscIO
+{
+void LaggedFibonacciGenerator::SetSeed(const u32 seed[SEED_SIZE])
+{
+  SetSeed(reinterpret_cast<const u8*>(seed));
+}
+
+void LaggedFibonacciGenerator::SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)])
+{
+  m_position_bytes = 0;
+
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+    m_buffer[i] = Common::swap32(seed + i * sizeof(u32));
+
+  Initialize(false);
+}
+
+size_t LaggedFibonacciGenerator::GetSeed(const u8* data, size_t size, size_t data_offset,
+                                         u32 seed_out[SEED_SIZE])
+{
+  if ((reinterpret_cast<uintptr_t>(data) - data_offset) % alignof(u32) != 0)
+  {
+    ASSERT(false);
+    return 0;
+  }
+
+  // For code simplicity, only include whole u32 words when regenerating the seed. It would be
+  // possible to get rid of this restriction and use a few additional bytes, but it's probably more
+  // effort than it's worth considering that junk data often starts or ends on 4-byte offsets.
+  const size_t bytes_to_skip = Common::AlignUp(data_offset, sizeof(u32)) - data_offset;
+  const u32* u32_data = reinterpret_cast<const u32*>(data + bytes_to_skip);
+  const size_t u32_size = (size - bytes_to_skip) / sizeof(u32);
+  const size_t u32_data_offset = (data_offset + bytes_to_skip) / sizeof(u32);
+
+  LaggedFibonacciGenerator lfg;
+  if (!GetSeed(u32_data, u32_size, u32_data_offset, &lfg, seed_out))
+    return false;
+
+  lfg.m_position_bytes = data_offset % (LFG_K * sizeof(u32));
+
+  const u8* end = data + size;
+  size_t reconstructed_bytes = 0;
+  while (data < end && lfg.GetByte() == *data)
+  {
+    ++reconstructed_bytes;
+    ++data;
+  }
+  return reconstructed_bytes;
+}
+
+bool LaggedFibonacciGenerator::GetSeed(const u32* data, size_t size, size_t data_offset,
+                                       LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE])
+{
+  if (size < LFG_K)
+    return false;
+
+  // If the data doesn't look like something we can regenerate, return early to save time
+  if (!std::all_of(data, data + LFG_K, [](u32 x) {
+        return (Common::swap32(x) & 0x00C00000) == (Common::swap32(x) >> 2 & 0x00C00000);
+      }))
+  {
+    return false;
+  }
+
+  const size_t data_offset_mod_k = data_offset % LFG_K;
+  const size_t data_offset_div_k = data_offset / LFG_K;
+
+  std::copy(data, data + LFG_K - data_offset_mod_k, lfg->m_buffer.data() + data_offset_mod_k);
+  std::copy(data + LFG_K - data_offset_mod_k, data + LFG_K, lfg->m_buffer.data());
+
+  lfg->Backward(0, data_offset_mod_k);
+
+  for (size_t i = 0; i < data_offset_div_k; ++i)
+    lfg->Backward();
+
+  if (!lfg->Reinitialize(seed_out))
+    return false;
+
+  for (size_t i = 0; i < data_offset_div_k; ++i)
+    lfg->Forward();
+
+  return true;
+}
+
+void LaggedFibonacciGenerator::GetBytes(size_t count, u8* out)
+{
+  while (count > 0)
+  {
+    const size_t length = std::min(count, LFG_K * sizeof(u32) - m_position_bytes);
+
+    std::memcpy(out, reinterpret_cast<u8*>(m_buffer.data()) + m_position_bytes, length);
+
+    m_position_bytes += length;
+    count -= length;
+    out += length;
+
+    if (m_position_bytes == LFG_K * sizeof(u32))
+    {
+      Forward();
+      m_position_bytes = 0;
+    }
+  }
+}
+
+u8 LaggedFibonacciGenerator::GetByte()
+{
+  const u8 result = reinterpret_cast<u8*>(m_buffer.data())[m_position_bytes];
+
+  ++m_position_bytes;
+
+  if (m_position_bytes == LFG_K * sizeof(u32))
+  {
+    Forward();
+    m_position_bytes = 0;
+  }
+
+  return result;
+}
+
+void LaggedFibonacciGenerator::Forward(size_t count)
+{
+  m_position_bytes += count;
+  while (m_position_bytes >= LFG_K * sizeof(u32))
+  {
+    Forward();
+    m_position_bytes -= LFG_K * sizeof(u32);
+  }
+}
+
+void LaggedFibonacciGenerator::Forward()
+{
+  for (size_t i = 0; i < LFG_J; ++i)
+    m_buffer[i] ^= m_buffer[i + LFG_K - LFG_J];
+
+  for (size_t i = LFG_J; i < LFG_K; ++i)
+    m_buffer[i] ^= m_buffer[i - LFG_J];
+}
+
+void LaggedFibonacciGenerator::Backward(size_t start_word, size_t end_word)
+{
+  const size_t loop_end = std::max(LFG_J, start_word);
+  for (size_t i = std::min(end_word, LFG_K); i > loop_end; --i)
+    m_buffer[i - 1] ^= m_buffer[i - 1 - LFG_J];
+
+  for (size_t i = std::min(end_word, LFG_J); i > start_word; --i)
+    m_buffer[i - 1] ^= m_buffer[i - 1 + LFG_K - LFG_J];
+}
+
+bool LaggedFibonacciGenerator::Reinitialize(u32 seed_out[SEED_SIZE])
+{
+  for (size_t i = 0; i < 4; ++i)
+    Backward();
+
+  for (u32& x : m_buffer)
+    x = Common::swap32(x);
+
+  // Reconstruct the bits which are missing due to the output code shifting by 18 instead of 16.
+  // Unfortunately we can't reconstruct bits 16 and 17 (counting LSB as 0) for the first word,
+  // but the observable result (when shifting by 18 instead of 16) is not affected by this.
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+  {
+    m_buffer[i] = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000) |
+                  ((m_buffer[i + 16] ^ m_buffer[i + 15]) << 9 & 0x00030000);
+  }
+
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+    seed_out[i] = Common::swap32(m_buffer[i]);
+
+  return Initialize(true);
+}
+
+bool LaggedFibonacciGenerator::Initialize(bool check_existing_data)
+{
+  for (size_t i = SEED_SIZE; i < LFG_K; ++i)
+  {
+    const u32 calculated = (m_buffer[i - 17] << 23) ^ (m_buffer[i - 16] >> 9) ^ m_buffer[i - 1];
+
+    if (check_existing_data)
+    {
+      const u32 actual = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000);
+      if ((calculated & 0xFFFCFFFF) != actual)
+        return false;
+    }
+
+    m_buffer[i] = calculated;
+  }
+
+  // Instead of doing the "shift by 18 instead of 16" oddity when actually outputting the data,
+  // we can do the shifting (and byteswapping) at this point to make the output code simpler.
+  for (u32& x : m_buffer)
+    x = Common::swap32((x & 0xFF00FFFF) | ((x >> 2) & 0x00FF0000));
+
+  for (size_t i = 0; i < 4; ++i)
+    Forward();
+
+  return true;
+}
+
+}  // namespace DiscIO

Source/Core/DiscIO/LaggedFibonacciGenerator.h

@@ -0,0 +1,51 @@
+// This file is under the public domain.
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+
+#include "Common/CommonTypes.h"
+
+namespace DiscIO
+{
+class LaggedFibonacciGenerator
+{
+public:
+  static constexpr size_t SEED_SIZE = 17;
+
+  // Reconstructs a seed and writes it to seed_out, then returns the number of bytes which can
+  // be reconstructed using that seed. Can return any number between 0 and size, inclusive.
+  // data - data_offset must be 4-byte aligned.
+  static size_t GetSeed(const u8* data, size_t size, size_t data_offset, u32 seed_out[SEED_SIZE]);
+
+  // SetSeed must be called before using the functions below
+  void SetSeed(const u32 seed[SEED_SIZE]);
+  void SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)]);
+
+  // Outputs a number of bytes and advances the internal state by the same amount.
+  void GetBytes(size_t count, u8* out);
+  u8 GetByte();
+
+  // Advances the internal state like GetBytes, but without outputting data. O(N), like GetBytes.
+  void Forward(size_t count);
+
+private:
+  static bool GetSeed(const u32* data, size_t size, size_t data_offset,
+                      LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE]);
+
+  void Forward();
+  void Backward(size_t start_word = 0, size_t end_word = LFG_K);
+
+  bool Reinitialize(u32 seed_out[SEED_SIZE]);
+  bool Initialize(bool check_existing_data);
+
+  static constexpr size_t LFG_K = 521;
+  static constexpr size_t LFG_J = 32;
+
+  std::array<u32, LFG_K> m_buffer;
+
+  size_t m_position_bytes = 0;
+};
+
+}  // namespace DiscIO

Source/Core/DiscIO/VolumeWii.cpp

@@ -201,18 +201,9 @@ bool VolumeWii::Read(u64 offset, u64 length, u8* buffer, const Partition& partit
       if (!m_reader->Read(block_offset_on_disc, BLOCK_TOTAL_SIZE, read_buffer.data()))
         return false;
 
-      // Decrypt the block's data.
-      // 0x3D0 - 0x3DF in read_buffer will be overwritten,
-      // but that won't affect anything, because we won't
-      // use the content of read_buffer anymore after this
-      mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, BLOCK_DATA_SIZE, &read_buffer[0x3D0],
-                            &read_buffer[BLOCK_HEADER_SIZE], m_last_decrypted_block_data);
+      // Decrypt the block's data
+      DecryptBlockData(read_buffer.data(), m_last_decrypted_block_data, aes_context);
       m_last_decrypted_block = block_offset_on_disc;
-
-      // The only thing we currently use from the 0x000 - 0x3FF part
-      // of the block is the IV (at 0x3D0), but it also contains SHA-1
-      // hashes that IOS uses to check that discs aren't tampered with.
-      // http://wiibrew.org/wiki/Wii_Disc#Encrypted
     }
 
     // Copy the decrypted data
@@ -482,14 +473,10 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const std::vector<u8>& encr
     return false;
 
   HashBlock hashes;
-  u8 iv[16] = {0};
-  mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(HashBlock), iv,
-                        encrypted_data.data(), reinterpret_cast<u8*>(&hashes));
+  DecryptBlockHashes(encrypted_data.data(), &hashes, aes_context);
 
   u8 cluster_data[BLOCK_DATA_SIZE];
-  std::memcpy(iv, encrypted_data.data() + 0x3D0, 16);
-  mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(cluster_data), iv,
-                        encrypted_data.data() + sizeof(HashBlock), cluster_data);
+  DecryptBlockData(encrypted_data.data(), cluster_data, aes_context);
 
   for (u32 hash_index = 0; hash_index < 31; ++hash_index)
   {
@@ -532,54 +519,33 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const Partition& partition)
   return CheckBlockIntegrity(block_index, cluster, partition);
 }
 
-bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset,
-                             u64 partition_data_decrypted_size,
-                             const std::array<u8, AES_KEY_SIZE>& key, BlobReader* blob,
-                             std::array<u8, GROUP_TOTAL_SIZE>* out)
+bool VolumeWii::HashGroup(const std::array<u8, BLOCK_DATA_SIZE> in[BLOCKS_PER_GROUP],
+                          HashBlock out[BLOCKS_PER_GROUP],
+                          const std::function<bool(size_t block)>& read_function)
 {
-  std::vector<std::array<u8, BLOCK_DATA_SIZE>> unencrypted_data(BLOCKS_PER_GROUP);
-  std::vector<HashBlock> unencrypted_hashes(BLOCKS_PER_GROUP);
-
   std::array<std::future<void>, BLOCKS_PER_GROUP> hash_futures;
-  bool error_occurred = false;
+  bool success = true;
 
   for (size_t i = 0; i < BLOCKS_PER_GROUP; ++i)
   {
-    if (!error_occurred)
-    {
-      if (offset + (i + 1) * BLOCK_DATA_SIZE <= partition_data_decrypted_size)
-      {
-        if (!blob->ReadWiiDecrypted(offset + i * BLOCK_DATA_SIZE, BLOCK_DATA_SIZE,
-                                    unencrypted_data[i].data(), partition_data_offset))
-        {
-          error_occurred = true;
-        }
-      }
-      else
-      {
-        unencrypted_data[i].fill(0);
-      }
-    }
+    if (read_function && success)
+      success = read_function(i);
 
-    hash_futures[i] = std::async(std::launch::async, [&unencrypted_data, &unencrypted_hashes,
-                                                      &hash_futures, error_occurred, i]() {
+    hash_futures[i] = std::async(std::launch::async, [&in, &out, &hash_futures, success, i]() {
       const size_t h1_base = Common::AlignDown(i, 8);
 
-      if (!error_occurred)
+      if (success)
       {
         // H0 hashes
         for (size_t j = 0; j < 31; ++j)
-        {
-          mbedtls_sha1_ret(unencrypted_data[i].data() + j * 0x400, 0x400,
-                           unencrypted_hashes[i].h0[j]);
-        }
+          mbedtls_sha1_ret(in[i].data() + j * 0x400, 0x400, out[i].h0[j]);
 
         // H0 padding
-        std::memset(unencrypted_hashes[i].padding_0, 0, sizeof(HashBlock::padding_0));
+        std::memset(out[i].padding_0, 0, sizeof(HashBlock::padding_0));
 
         // H1 hash
-        mbedtls_sha1_ret(reinterpret_cast<u8*>(unencrypted_hashes[i].h0), sizeof(HashBlock::h0),
-                         unencrypted_hashes[h1_base].h1[i - h1_base]);
+        mbedtls_sha1_ret(reinterpret_cast<u8*>(out[i].h0), sizeof(HashBlock::h0),
+                         out[h1_base].h1[i - h1_base]);
       }
 
       if (i % 8 == 7)
@@ -587,21 +553,18 @@ bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset,
         for (size_t j = 0; j < 7; ++j)
           hash_futures[h1_base + j].get();
 
-        if (!error_occurred)
+        if (success)
         {
           // H1 padding
-          std::memset(unencrypted_hashes[h1_base].padding_1, 0, sizeof(HashBlock::padding_1));
+          std::memset(out[h1_base].padding_1, 0, sizeof(HashBlock::padding_1));
 
           // H1 copies
           for (size_t j = 1; j < 8; ++j)
-          {
-            std::memcpy(unencrypted_hashes[h1_base + j].h1, unencrypted_hashes[h1_base].h1,
-                        sizeof(HashBlock::h1));
-          }
+            std::memcpy(out[h1_base + j].h1, out[h1_base].h1, sizeof(HashBlock::h1));
 
           // H2 hash
-          mbedtls_sha1_ret(reinterpret_cast<u8*>(unencrypted_hashes[i].h1), sizeof(HashBlock::h1),
-                           unencrypted_hashes[0].h2[h1_base / 8]);
+          mbedtls_sha1_ret(reinterpret_cast<u8*>(out[i].h1), sizeof(HashBlock::h1),
+                           out[0].h2[h1_base / 8]);
         }
 
         if (i == BLOCKS_PER_GROUP - 1)
@@ -609,17 +572,14 @@ bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset,
           for (size_t j = 0; j < 7; ++j)
             hash_futures[j * 8 + 7].get();
 
-          if (!error_occurred)
+          if (success)
           {
             // H2 padding
-            std::memset(unencrypted_hashes[0].padding_2, 0, sizeof(HashBlock::padding_2));
+            std::memset(out[0].padding_2, 0, sizeof(HashBlock::padding_2));
 
             // H2 copies
             for (size_t j = 1; j < BLOCKS_PER_GROUP; ++j)
-            {
-              std::memcpy(unencrypted_hashes[j].h2, unencrypted_hashes[0].h2,
-                          sizeof(HashBlock::h2));
-            }
+              std::memcpy(out[j].h2, out[0].h2, sizeof(HashBlock::h2));
           }
         }
       }
@@ -629,9 +589,41 @@ bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset,
   // Wait for all the async tasks to finish
   hash_futures.back().get();
 
-  if (error_occurred)
+  return success;
+}
+
+bool VolumeWii::EncryptGroup(
+    u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size,
+    const std::array<u8, AES_KEY_SIZE>& key, BlobReader* blob,
+    std::array<u8, GROUP_TOTAL_SIZE>* out,
+    const std::function<void(HashBlock hash_blocks[BLOCKS_PER_GROUP])>& hash_exception_callback)
+{
+  std::vector<std::array<u8, BLOCK_DATA_SIZE>> unencrypted_data(BLOCKS_PER_GROUP);
+  std::vector<HashBlock> unencrypted_hashes(BLOCKS_PER_GROUP);
+
+  const bool success =
+      HashGroup(unencrypted_data.data(), unencrypted_hashes.data(), [&](size_t block) {
+        if (offset + (block + 1) * BLOCK_DATA_SIZE <= partition_data_decrypted_size)
+        {
+          if (!blob->ReadWiiDecrypted(offset + block * BLOCK_DATA_SIZE, BLOCK_DATA_SIZE,
+                                      unencrypted_data[block].data(), partition_data_offset))
+          {
+            return false;
+          }
+        }
+        else
+        {
+          unencrypted_data[block].fill(0);
+        }
+        return true;
+      });
+
+  if (!success)
     return false;
 
+  if (hash_exception_callback)
+    hash_exception_callback(unencrypted_hashes.data());
+
   const unsigned int threads =
       std::min(BLOCKS_PER_GROUP, std::max<unsigned int>(1, std::thread::hardware_concurrency()));
 
@@ -667,4 +659,20 @@ bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset,
   return true;
 }
 
+void VolumeWii::DecryptBlockHashes(const u8* in, HashBlock* out, mbedtls_aes_context* aes_context)
+{
+  std::array<u8, 16> iv;
+  iv.fill(0);
+  mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(HashBlock), iv.data(), in,
+                        reinterpret_cast<u8*>(out));
+}
+
+void VolumeWii::DecryptBlockData(const u8* in, u8* out, mbedtls_aes_context* aes_context)
+{
+  std::array<u8, 16> iv;
+  std::copy(&in[0x3d0], &in[0x3e0], iv.data());
+  mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, BLOCK_DATA_SIZE, iv.data(),
+                        &in[BLOCK_HEADER_SIZE], out);
+}
+
 }  // namespace DiscIO

Source/Core/DiscIO/VolumeWii.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include <array>
+#include <functional>
 #include <map>
 #include <memory>
 #include <optional>
@@ -97,9 +98,22 @@ public:
   u64 GetRawSize() const override;
   const BlobReader& GetBlobReader() const;
 
+  // The in parameter can either contain all the data to begin with,
+  // or read_function can write data into the in parameter when called.
+  // The latter lets reading run in parallel with hashing.
+  // This function returns false iff read_function returns false.
+  static bool HashGroup(const std::array<u8, BLOCK_DATA_SIZE> in[BLOCKS_PER_GROUP],
+                        HashBlock out[BLOCKS_PER_GROUP],
+                        const std::function<bool(size_t block)>& read_function = {});
+
   static bool EncryptGroup(u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size,
                            const std::array<u8, AES_KEY_SIZE>& key, BlobReader* blob,
-                           std::array<u8, GROUP_TOTAL_SIZE>* out);
+                           std::array<u8, GROUP_TOTAL_SIZE>* out,
+                           const std::function<void(HashBlock hash_blocks[BLOCKS_PER_GROUP])>&
+                               hash_exception_callback = {});
+
+  static void DecryptBlockHashes(const u8* in, HashBlock* out, mbedtls_aes_context* aes_context);
+  static void DecryptBlockData(const u8* in, u8* out, mbedtls_aes_context* aes_context);
 
 protected:
   u32 GetOffsetShift() const override { return 2; }

Source/Core/DiscIO/WIABlob.h

@@ -0,0 +1,390 @@
+// Copyright 2018 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <array>
+#include <limits>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <type_traits>
+#include <utility>
+
+#include "Common/CommonTypes.h"
+#include "Common/File.h"
+#include "Common/Swap.h"
+#include "DiscIO/Blob.h"
+#include "DiscIO/MultithreadedCompressor.h"
+#include "DiscIO/WIACompression.h"
+#include "DiscIO/WiiEncryptionCache.h"
+
+namespace DiscIO
+{
+class FileSystem;
+class VolumeDisc;
+
+enum class WIARVZCompressionType : u32
+{
+  None = 0,
+  Purge = 1,
+  Bzip2 = 2,
+  LZMA = 3,
+  LZMA2 = 4,
+  Zstd = 5,
+};
+
+std::pair<int, int> GetAllowedCompressionLevels(WIARVZCompressionType compression_type);
+
+constexpr u32 WIA_MAGIC = 0x01414957;  // "WIA\x1" (byteswapped to little endian)
+constexpr u32 RVZ_MAGIC = 0x015A5652;  // "RVZ\x1" (byteswapped to little endian)
+
+template <bool RVZ>
+class WIARVZFileReader : public BlobReader
+{
+public:
+  ~WIARVZFileReader();
+
+  static std::unique_ptr<WIARVZFileReader> Create(File::IOFile file, const std::string& path);
+
+  BlobType GetBlobType() const override;
+
+  u64 GetRawSize() const override { return Common::swap64(m_header_1.wia_file_size); }
+  u64 GetDataSize() const override { return Common::swap64(m_header_1.iso_file_size); }
+  bool IsDataSizeAccurate() const override { return true; }
+
+  u64 GetBlockSize() const override { return Common::swap32(m_header_2.chunk_size); }
+  bool HasFastRandomAccessInBlock() const override { return false; }
+
+  bool Read(u64 offset, u64 size, u8* out_ptr) override;
+  bool SupportsReadWiiDecrypted() const override;
+  bool ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) override;
+
+  static ConversionResultCode Convert(BlobReader* infile, const VolumeDisc* infile_volume,
+                                      File::IOFile* outfile, WIARVZCompressionType compression_type,
+                                      int compression_level, int chunk_size, CompressCB callback,
+                                      void* arg);
+
+private:
+  using SHA1 = std::array<u8, 20>;
+  using WiiKey = std::array<u8, 16>;
+
+  // See docs/WIA.md for details about the format
+
+#pragma pack(push, 1)
+  struct WIAHeader1
+  {
+    u32 magic;
+    u32 version;
+    u32 version_compatible;
+    u32 header_2_size;
+    SHA1 header_2_hash;
+    u64 iso_file_size;
+    u64 wia_file_size;
+    SHA1 header_1_hash;
+  };
+  static_assert(sizeof(WIAHeader1) == 0x48, "Wrong size for WIA header 1");
+
+  struct WIAHeader2
+  {
+    u32 disc_type;
+    u32 compression_type;
+    u32 compression_level;  // Informative only
+    u32 chunk_size;
+
+    std::array<u8, 0x80> disc_header;
+
+    u32 number_of_partition_entries;
+    u32 partition_entry_size;
+    u64 partition_entries_offset;
+    SHA1 partition_entries_hash;
+
+    u32 number_of_raw_data_entries;
+    u64 raw_data_entries_offset;
+    u32 raw_data_entries_size;
+
+    u32 number_of_group_entries;
+    u64 group_entries_offset;
+    u32 group_entries_size;
+
+    u8 compressor_data_size;
+    u8 compressor_data[7];
+  };
+  static_assert(sizeof(WIAHeader2) == 0xdc, "Wrong size for WIA header 2");
+
+  struct PartitionDataEntry
+  {
+    u32 first_sector;
+    u32 number_of_sectors;
+    u32 group_index;
+    u32 number_of_groups;
+  };
+  static_assert(sizeof(PartitionDataEntry) == 0x10, "Wrong size for WIA partition data entry");
+
+  struct PartitionEntry
+  {
+    WiiKey partition_key;
+    std::array<PartitionDataEntry, 2> data_entries;
+  };
+  static_assert(sizeof(PartitionEntry) == 0x30, "Wrong size for WIA partition entry");
+
+  struct RawDataEntry
+  {
+    u64 data_offset;
+    u64 data_size;
+    u32 group_index;
+    u32 number_of_groups;
+  };
+  static_assert(sizeof(RawDataEntry) == 0x18, "Wrong size for WIA raw data entry");
+
+  struct WIAGroupEntry
+  {
+    u32 data_offset;  // >> 2
+    u32 data_size;
+  };
+  static_assert(sizeof(WIAGroupEntry) == 0x08, "Wrong size for WIA group entry");
+
+  struct RVZGroupEntry
+  {
+    u32 data_offset;  // >> 2
+    u32 data_size;
+    u32 rvz_packed_size;
+  };
+  static_assert(sizeof(RVZGroupEntry) == 0x0c, "Wrong size for RVZ group entry");
+
+  using GroupEntry = std::conditional_t<RVZ, RVZGroupEntry, WIAGroupEntry>;
+
+  struct HashExceptionEntry
+  {
+    u16 offset;
+    SHA1 hash;
+  };
+  static_assert(sizeof(HashExceptionEntry) == 0x16, "Wrong size for WIA hash exception entry");
+#pragma pack(pop)
+
+  struct DataEntry
+  {
+    u32 index;
+    bool is_partition;
+    u8 partition_data_index;
+
+    DataEntry(size_t index_) : index(static_cast<u32>(index_)), is_partition(false) {}
+    DataEntry(size_t index_, size_t partition_data_index_)
+        : index(static_cast<u32>(index_)), is_partition(true),
+          partition_data_index(static_cast<u8>(partition_data_index_))
+    {
+    }
+  };
+
+  class Chunk
+  {
+  public:
+    Chunk();
+    Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size,
+          u32 exception_lists, bool compressed_exception_lists, u32 rvz_packed_size,
+          u64 data_offset, std::unique_ptr<Decompressor> decompressor);
+
+    bool Read(u64 offset, u64 size, u8* out_ptr);
+
+    // This can only be called once at least one byte of data has been read
+    void GetHashExceptions(std::vector<HashExceptionEntry>* exception_list,
+                           u64 exception_list_index, u16 additional_offset) const;
+
+    template <typename T>
+    bool ReadAll(std::vector<T>* vector)
+    {
+      return Read(0, vector->size() * sizeof(T), reinterpret_cast<u8*>(vector->data()));
+    }
+
+  private:
+    bool Decompress();
+    bool HandleExceptions(const u8* data, size_t bytes_allocated, size_t bytes_written,
+                          size_t* bytes_used, bool align);
+
+    DecompressionBuffer m_in;
+    DecompressionBuffer m_out;
+    size_t m_in_bytes_read = 0;
+
+    std::unique_ptr<Decompressor> m_decompressor = nullptr;
+    File::IOFile* m_file = nullptr;
+    u64 m_offset_in_file = 0;
+
+    size_t m_out_bytes_allocated_for_exceptions = 0;
+    size_t m_out_bytes_used_for_exceptions = 0;
+    size_t m_in_bytes_used_for_exceptions = 0;
+    u32 m_exception_lists = 0;
+    bool m_compressed_exception_lists = false;
+    u32 m_rvz_packed_size = 0;
+    u64 m_data_offset = 0;
+  };
+
+  explicit WIARVZFileReader(File::IOFile file, const std::string& path);
+  bool Initialize(const std::string& path);
+  bool HasDataOverlap() const;
+
+  bool ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, u32 sector_size,
+                      u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups,
+                      u32 exception_lists);
+  Chunk& ReadCompressedData(u64 offset_in_file, u64 compressed_size, u64 decompressed_size,
+                            WIARVZCompressionType compression_type, u32 exception_lists = 0,
+                            u32 rvz_packed_size = 0, u64 data_offset = 0);
+
+  static bool ApplyHashExceptions(const std::vector<HashExceptionEntry>& exception_list,
+                                  VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]);
+
+  static std::string VersionToString(u32 version);
+
+  struct ReuseID
+  {
+    bool operator==(const ReuseID& other) const
+    {
+      return std::tie(partition_key, data_size, encrypted, value) ==
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
+    }
+    bool operator<(const ReuseID& other) const
+    {
+      return std::tie(partition_key, data_size, encrypted, value) <
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
+    }
+    bool operator>(const ReuseID& other) const
+    {
+      return std::tie(partition_key, data_size, encrypted, value) >
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
+    }
+    bool operator!=(const ReuseID& other) const { return !operator==(other); }
+    bool operator>=(const ReuseID& other) const { return !operator<(other); }
+    bool operator<=(const ReuseID& other) const { return !operator>(other); }
+
+    const WiiKey* partition_key;
+    u64 data_size;
+    bool encrypted;
+    u8 value;
+  };
+
+  struct CompressThreadState
+  {
+    using WiiBlockData = std::array<u8, VolumeWii::BLOCK_DATA_SIZE>;
+
+    std::unique_ptr<Compressor> compressor;
+
+    std::vector<WiiBlockData> decryption_buffer =
+        std::vector<WiiBlockData>(VolumeWii::BLOCKS_PER_GROUP);
+
+    std::vector<VolumeWii::HashBlock> hash_buffer =
+        std::vector<VolumeWii::HashBlock>(VolumeWii::BLOCKS_PER_GROUP);
+  };
+
+  struct CompressParameters
+  {
+    std::vector<u8> data;
+    const DataEntry* data_entry;
+    u64 data_offset;
+    u64 bytes_read;
+    size_t group_index;
+  };
+
+  struct WIAOutputParametersEntry
+  {
+    std::vector<u8> exception_lists;
+    std::vector<u8> main_data;
+    std::optional<ReuseID> reuse_id;
+    std::optional<GroupEntry> reused_group;
+  };
+
+  struct RVZOutputParametersEntry
+  {
+    std::vector<u8> exception_lists;
+    std::vector<u8> main_data;
+    std::optional<ReuseID> reuse_id;
+    std::optional<GroupEntry> reused_group;
+    size_t rvz_packed_size = 0;
+    bool compressed = false;
+  };
+
+  using OutputParametersEntry =
+      std::conditional_t<RVZ, RVZOutputParametersEntry, WIAOutputParametersEntry>;
+
+  struct OutputParameters
+  {
+    std::vector<OutputParametersEntry> entries;
+    u64 bytes_read;
+    size_t group_index;
+  };
+
+  static bool PadTo4(File::IOFile* file, u64* bytes_written);
+  static void AddRawDataEntry(u64 offset, u64 size, int chunk_size, u32* total_groups,
+                              std::vector<RawDataEntry>* raw_data_entries,
+                              std::vector<DataEntry>* data_entries);
+  static PartitionDataEntry
+  CreatePartitionDataEntry(u64 offset, u64 size, u32 index, int chunk_size, u32* total_groups,
+                           const std::vector<PartitionEntry>& partition_entries,
+                           std::vector<DataEntry>* data_entries);
+  static ConversionResultCode SetUpDataEntriesForWriting(
+      const VolumeDisc* volume, int chunk_size, u64 iso_size, u32* total_groups,
+      std::vector<PartitionEntry>* partition_entries, std::vector<RawDataEntry>* raw_data_entries,
+      std::vector<DataEntry>* data_entries, std::vector<const FileSystem*>* partition_file_systems);
+  static std::optional<std::vector<u8>> Compress(Compressor* compressor, const u8* data,
+                                                 size_t size);
+  static bool WriteHeader(File::IOFile* file, const u8* data, size_t size, u64 upper_bound,
+                          u64* bytes_written, u64* offset_out);
+
+  static void SetUpCompressor(std::unique_ptr<Compressor>* compressor,
+                              WIARVZCompressionType compression_type, int compression_level,
+                              WIAHeader2* header_2);
+  static bool TryReuse(std::map<ReuseID, GroupEntry>* reusable_groups,
+                       std::mutex* reusable_groups_mutex, OutputParametersEntry* entry);
+  static ConversionResult<OutputParameters>
+  ProcessAndCompress(CompressThreadState* state, CompressParameters parameters,
+                     const std::vector<PartitionEntry>& partition_entries,
+                     const std::vector<DataEntry>& data_entries, const FileSystem* file_system,
+                     std::map<ReuseID, GroupEntry>* reusable_groups,
+                     std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group,
+                     u64 exception_lists_per_chunk, bool compressed_exception_lists,
+                     bool compression);
+  static ConversionResultCode Output(std::vector<OutputParametersEntry>* entries,
+                                     File::IOFile* outfile,
+                                     std::map<ReuseID, GroupEntry>* reusable_groups,
+                                     std::mutex* reusable_groups_mutex, GroupEntry* group_entry,
+                                     u64* bytes_written);
+  static ConversionResultCode RunCallback(size_t groups_written, u64 bytes_read, u64 bytes_written,
+                                          u32 total_groups, u64 iso_size, CompressCB callback,
+                                          void* arg);
+
+  bool m_valid;
+  WIARVZCompressionType m_compression_type;
+
+  File::IOFile m_file;
+  Chunk m_cached_chunk;
+  u64 m_cached_chunk_offset = std::numeric_limits<u64>::max();
+  WiiEncryptionCache m_encryption_cache;
+
+  std::vector<HashExceptionEntry> m_exception_list;
+  bool m_write_to_exception_list = false;
+  u64 m_exception_list_last_group_index;
+
+  WIAHeader1 m_header_1;
+  WIAHeader2 m_header_2;
+  std::vector<PartitionEntry> m_partition_entries;
+  std::vector<RawDataEntry> m_raw_data_entries;
+  std::vector<GroupEntry> m_group_entries;
+
+  std::map<u64, DataEntry> m_data_entries;
+
+  // Perhaps we could set WIA_VERSION_WRITE_COMPATIBLE to 0.9, but WIA version 0.9 was never in
+  // any official release of wit, and interim versions (either source or binaries) are hard to find.
+  // Since we've been unable to check if we're write compatible with 0.9, we set it 1.0 to be safe.
+
+  static constexpr u32 WIA_VERSION = 0x01000000;
+  static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000;
+  static constexpr u32 WIA_VERSION_READ_COMPATIBLE = 0x00080000;
+
+  static constexpr u32 RVZ_VERSION = 0x01000000;
+  static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00030000;
+  static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00030000;
+};
+
+using WIAFileReader = WIARVZFileReader<false>;
+using RVZFileReader = WIARVZFileReader<true>;
+
+}  // namespace DiscIO

Source/Core/DiscIO/WIACompression.h

@@ -0,0 +1,252 @@
+// Copyright 2020 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <cstddef>
+#include <memory>
+#include <optional>
+#include <vector>
+
+#include <bzlib.h>
+#include <lzma.h>
+#include <mbedtls/sha1.h>
+#include <zstd.h>
+
+#include "Common/CommonTypes.h"
+#include "DiscIO/LaggedFibonacciGenerator.h"
+
+namespace DiscIO
+{
+struct DecompressionBuffer
+{
+  std::vector<u8> data;
+  size_t bytes_written = 0;
+};
+
+using SHA1 = std::array<u8, 20>;
+
+struct PurgeSegment
+{
+  u32 offset;
+  u32 size;
+};
+static_assert(sizeof(PurgeSegment) == 0x08, "Wrong size for WIA purge segment");
+
+class Decompressor
+{
+public:
+  virtual ~Decompressor();
+
+  virtual bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                          size_t* in_bytes_read) = 0;
+  virtual bool Done() const { return m_done; };
+
+protected:
+  bool m_done = false;
+};
+
+class NoneDecompressor final : public Decompressor
+{
+public:
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+};
+
+// This class assumes that more bytes won't be added to in once in.bytes_written == in.data.size()
+// and that *in_bytes_read initially will be equal to the size of the exception lists
+class PurgeDecompressor final : public Decompressor
+{
+public:
+  PurgeDecompressor(u64 decompressed_size);
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+
+private:
+  const u64 m_decompressed_size;
+
+  PurgeSegment m_segment = {};
+  size_t m_bytes_read = 0;
+  size_t m_segment_bytes_written = 0;
+  size_t m_out_bytes_written = 0;
+  bool m_started = false;
+
+  mbedtls_sha1_context m_sha1_context;
+};
+
+class Bzip2Decompressor final : public Decompressor
+{
+public:
+  ~Bzip2Decompressor();
+
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+
+private:
+  bz_stream m_stream = {};
+  bool m_started = false;
+};
+
+class LZMADecompressor final : public Decompressor
+{
+public:
+  LZMADecompressor(bool lzma2, const u8* filter_options, size_t filter_options_size);
+  ~LZMADecompressor();
+
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+
+private:
+  lzma_stream m_stream = LZMA_STREAM_INIT;
+  lzma_options_lzma m_options = {};
+  lzma_filter m_filters[2];
+  bool m_started = false;
+  bool m_error_occurred = false;
+};
+
+class ZstdDecompressor final : public Decompressor
+{
+public:
+  ZstdDecompressor();
+  ~ZstdDecompressor();
+
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+
+private:
+  ZSTD_DStream* m_stream;
+};
+
+class RVZPackDecompressor final : public Decompressor
+{
+public:
+  RVZPackDecompressor(std::unique_ptr<Decompressor> decompressor, DecompressionBuffer decompressed,
+                      u64 data_offset, u32 rvz_packed_size);
+
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+
+  bool Done() const override;
+
+private:
+  bool IncrementBytesRead(size_t x);
+  std::optional<bool> ReadToDecompressed(const DecompressionBuffer& in, size_t* in_bytes_read,
+                                         size_t decompressed_bytes_read, size_t bytes_to_read);
+
+  std::unique_ptr<Decompressor> m_decompressor;
+  DecompressionBuffer m_decompressed;
+  size_t m_decompressed_bytes_read = 0;
+  size_t m_bytes_read;
+  u64 m_data_offset;
+  u32 m_rvz_packed_size;
+
+  u32 m_size = 0;
+  bool m_junk;
+  LaggedFibonacciGenerator m_lfg;
+};
+
+class Compressor
+{
+public:
+  virtual ~Compressor();
+
+  // First call Start, then AddDataOnlyForPurgeHashing/Compress any number of times,
+  // then End, then GetData/GetSize any number of times.
+
+  virtual bool Start() = 0;
+  virtual bool AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) { return true; }
+  virtual bool Compress(const u8* data, size_t size) = 0;
+  virtual bool End() = 0;
+
+  virtual const u8* GetData() const = 0;
+  virtual size_t GetSize() const = 0;
+};
+
+class PurgeCompressor final : public Compressor
+{
+public:
+  PurgeCompressor();
+  ~PurgeCompressor();
+
+  bool Start() override;
+  bool AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) override;
+  bool Compress(const u8* data, size_t size) override;
+  bool End() override;
+
+  const u8* GetData() const override;
+  size_t GetSize() const override;
+
+private:
+  std::vector<u8> m_buffer;
+  size_t m_bytes_written;
+  mbedtls_sha1_context m_sha1_context;
+};
+
+class Bzip2Compressor final : public Compressor
+{
+public:
+  Bzip2Compressor(int compression_level);
+  ~Bzip2Compressor();
+
+  bool Start() override;
+  bool Compress(const u8* data, size_t size) override;
+  bool End() override;
+
+  const u8* GetData() const override;
+  size_t GetSize() const override;
+
+private:
+  void ExpandBuffer(size_t bytes_to_add);
+
+  bz_stream m_stream = {};
+  std::vector<u8> m_buffer;
+  int m_compression_level;
+};
+
+class LZMACompressor final : public Compressor
+{
+public:
+  LZMACompressor(bool lzma2, int compression_level, u8 compressor_data_out[7],
+                 u8* compressor_data_size_out);
+  ~LZMACompressor();
+
+  bool Start() override;
+  bool Compress(const u8* data, size_t size) override;
+  bool End() override;
+
+  const u8* GetData() const override;
+  size_t GetSize() const override;
+
+private:
+  void ExpandBuffer(size_t bytes_to_add);
+
+  lzma_stream m_stream = LZMA_STREAM_INIT;
+  lzma_options_lzma m_options = {};
+  lzma_filter m_filters[2];
+  std::vector<u8> m_buffer;
+  bool m_initialization_failed = false;
+};
+
+class ZstdCompressor final : public Compressor
+{
+public:
+  ZstdCompressor(int compression_level);
+  ~ZstdCompressor();
+
+  bool Start() override;
+  bool Compress(const u8* data, size_t size) override;
+  bool End() override;
+
+  const u8* GetData() const override { return m_buffer.data(); }
+  size_t GetSize() const override { return m_out_buffer.pos; }
+
+private:
+  void ExpandBuffer(size_t bytes_to_add);
+
+  ZSTD_CStream* m_stream;
+  ZSTD_outBuffer m_out_buffer;
+  std::vector<u8> m_buffer;
+};
+
+}  // namespace DiscIO

Source/Core/DiscIO/WiiEncryptionCache.cpp

@@ -24,7 +24,8 @@ WiiEncryptionCache::~WiiEncryptionCache() = default;
 
 const std::array<u8, VolumeWii::GROUP_TOTAL_SIZE>*
 WiiEncryptionCache::EncryptGroup(u64 offset, u64 partition_data_offset,
-                                 u64 partition_data_decrypted_size, const Key& key)
+                                 u64 partition_data_decrypted_size, const Key& key,
+                                 const HashExceptionCallback& hash_exception_callback)
 {
   // Only allocate memory if this function actually ends up getting called
   if (!m_cache)
@@ -40,8 +41,20 @@ WiiEncryptionCache::EncryptGroup(u64 offset, u64 partition_data_offset,
 
   if (m_cached_offset != group_offset_on_disc)
   {
+    std::function<void(VolumeWii::HashBlock * hash_blocks)> hash_exception_callback_2;
+
+    if (hash_exception_callback)
+    {
+      hash_exception_callback_2 =
+          [offset, &hash_exception_callback](
+              VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]) {
+            return hash_exception_callback(hash_blocks, offset);
+          };
+    }
+
     if (!VolumeWii::EncryptGroup(group_offset_in_partition, partition_data_offset,
-                                 partition_data_decrypted_size, key, m_blob, m_cache.get()))
+                                 partition_data_decrypted_size, key, m_blob, m_cache.get(),
+                                 hash_exception_callback_2))
     {
       m_cached_offset = std::numeric_limits<u64>::max();  // Invalidate the cache
       return nullptr;
@@ -54,13 +67,14 @@ WiiEncryptionCache::EncryptGroup(u64 offset, u64 partition_data_offset,
 }
 
 bool WiiEncryptionCache::EncryptGroups(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset,
-                                       u64 partition_data_decrypted_size, const Key& key)
+                                       u64 partition_data_decrypted_size, const Key& key,
+                                       const HashExceptionCallback& hash_exception_callback)
 {
   while (size > 0)
   {
     const std::array<u8, VolumeWii::GROUP_TOTAL_SIZE>* group =
         EncryptGroup(Common::AlignDown(offset, VolumeWii::GROUP_TOTAL_SIZE), partition_data_offset,
-                     partition_data_decrypted_size, key);
+                     partition_data_decrypted_size, key, hash_exception_callback);
 
     if (!group)
       return false;

Source/Core/DiscIO/WiiEncryptionCache.h

@@ -19,6 +19,8 @@ class WiiEncryptionCache
 {
 public:
   using Key = std::array<u8, VolumeWii::AES_KEY_SIZE>;
+  using HashExceptionCallback = std::function<void(
+      VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP], u64 offset)>;
 
   // The blob pointer is kept around for the lifetime of this object.
   explicit WiiEncryptionCache(BlobReader* blob);
@@ -28,15 +30,15 @@ public:
   // If the returned pointer is nullptr, reading from the blob failed.
   // If the returned pointer is not nullptr, it is guaranteed to be valid until
   // the next call of this function or the destruction of this object.
-  const std::array<u8, VolumeWii::GROUP_TOTAL_SIZE>* EncryptGroup(u64 offset,
-                                                                  u64 partition_data_offset,
-                                                                  u64 partition_data_decrypted_size,
-                                                                  const Key& key);
+  const std::array<u8, VolumeWii::GROUP_TOTAL_SIZE>*
+  EncryptGroup(u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size,
+               const Key& key, const HashExceptionCallback& hash_exception_callback = {});
 
   // Encrypts a variable number of groups, as determined by the offset and size parameters.
   // Supports reading groups partially.
   bool EncryptGroups(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset,
-                     u64 partition_data_decrypted_size, const Key& key);
+                     u64 partition_data_decrypted_size, const Key& key,
+                     const HashExceptionCallback& hash_exception_callback = {});
 
 private:
   BlobReader* m_blob;

Source/Core/DolphinQt/ConvertDialog.cpp

@@ -26,6 +26,7 @@
 #include "Common/Logging/Log.h"
 #include "DiscIO/Blob.h"
 #include "DiscIO/ScrubbedBlob.h"
+#include "DiscIO/WIABlob.h"
 #include "DolphinQt/QtUtils/ModalMessageBox.h"
 #include "DolphinQt/QtUtils/ParallelProgressDialog.h"
 #include "UICommon/GameFile.h"
@@ -57,6 +58,8 @@ ConvertDialog::ConvertDialog(QList<std::shared_ptr<const UICommon::GameFile>> fi
   m_format = new QComboBox;
   m_format->addItem(QStringLiteral("ISO"), static_cast<int>(DiscIO::BlobType::PLAIN));
   m_format->addItem(QStringLiteral("GCZ"), static_cast<int>(DiscIO::BlobType::GCZ));
+  m_format->addItem(QStringLiteral("WIA"), static_cast<int>(DiscIO::BlobType::WIA));
+  m_format->addItem(QStringLiteral("RVZ"), static_cast<int>(DiscIO::BlobType::RVZ));
   if (std::all_of(m_files.begin(), m_files.end(),
                   [](const auto& file) { return file->GetBlobType() == DiscIO::BlobType::PLAIN; }))
   {
@@ -69,9 +72,17 @@ ConvertDialog::ConvertDialog(QList<std::shared_ptr<const UICommon::GameFile>> fi
   grid_layout->addWidget(new QLabel(tr("Block Size:")), 1, 0);
   grid_layout->addWidget(m_block_size, 1, 1);
 
+  m_compression = new QComboBox;
+  grid_layout->addWidget(new QLabel(tr("Compression:")), 2, 0);
+  grid_layout->addWidget(m_compression, 2, 1);
+
+  m_compression_level = new QComboBox;
+  grid_layout->addWidget(new QLabel(tr("Compression Level:")), 3, 0);
+  grid_layout->addWidget(m_compression_level, 3, 1);
+
   m_scrub = new QCheckBox;
-  grid_layout->addWidget(new QLabel(tr("Remove Junk Data (Irreversible):")), 2, 0);
-  grid_layout->addWidget(m_scrub, 2, 1);
+  grid_layout->addWidget(new QLabel(tr("Remove Junk Data (Irreversible):")), 4, 0);
+  grid_layout->addWidget(m_scrub, 4, 1);
   m_scrub->setEnabled(
       std::none_of(m_files.begin(), m_files.end(), std::mem_fn(&UICommon::GameFile::IsDatelDisc)));
 
@@ -83,12 +94,17 @@ ConvertDialog::ConvertDialog(QList<std::shared_ptr<const UICommon::GameFile>> fi
   QGroupBox* options_group = new QGroupBox(tr("Options"));
   options_group->setLayout(options_layout);
 
-  QLabel* info_text =
-      new QLabel(tr("ISO: A simple and robust format which is supported by many programs. "
-                    "It takes up more space than any other format.\n\n"
-                    "GCZ: A basic compressed format which is compatible with most versions of "
-                    "Dolphin and some other programs. It can't efficiently compress junk data "
-                    "(unless removed) or encrypted Wii data."));
+  QLabel* info_text = new QLabel(
+      tr("ISO: A simple and robust format which is supported by many programs. It takes up more "
+         "space than any other format.\n\n"
+         "GCZ: A basic compressed format which is compatible with most versions of Dolphin and "
+         "some other programs. It can't efficiently compress junk data (unless removed) or "
+         "encrypted Wii data.\n\n"
+         "WIA: An advanced compressed format which is compatible with recent versions of Dolphin "
+         "and a few other programs. It can efficiently compress encrypted Wii data, but not junk "
+         "data (unless removed).\n\n"
+         "RVZ: An advanced compressed format which is compatible with recent versions of Dolphin. "
+         "It can efficiently compress both junk data and encrypted Wii data."));
   info_text->setWordWrap(true);
 
   QVBoxLayout* info_layout = new QVBoxLayout;
@@ -104,14 +120,34 @@ ConvertDialog::ConvertDialog(QList<std::shared_ptr<const UICommon::GameFile>> fi
 
   connect(m_format, QOverload<int>::of(&QComboBox::currentIndexChanged), this,
           &ConvertDialog::OnFormatChanged);
+  connect(m_compression, QOverload<int>::of(&QComboBox::currentIndexChanged), this,
+          &ConvertDialog::OnCompressionChanged);
   connect(convert_button, &QPushButton::clicked, this, &ConvertDialog::Convert);
 
   OnFormatChanged();
+  OnCompressionChanged();
 }
 
 void ConvertDialog::AddToBlockSizeComboBox(int size)
 {
   m_block_size->addItem(QString::fromStdString(UICommon::FormatSize(size, 0)), size);
+
+  // Select 128 KiB by default, or if it is not available, the size closest to it.
+  // This code assumes that sizes get added to the combo box in increasing order.
+  constexpr int DEFAULT_SIZE = 0x20000;
+  if (size <= DEFAULT_SIZE)
+    m_block_size->setCurrentIndex(m_block_size->count() - 1);
+}
+
+void ConvertDialog::AddToCompressionComboBox(const QString& name,
+                                             DiscIO::WIARVZCompressionType type)
+{
+  m_compression->addItem(name, static_cast<int>(type));
+}
+
+void ConvertDialog::AddToCompressionLevelComboBox(int level)
+{
+  m_compression_level->addItem(QString::number(level), level);
 }
 
 void ConvertDialog::OnFormatChanged()
@@ -127,6 +163,9 @@ void ConvertDialog::OnFormatChanged()
   const DiscIO::BlobType format = static_cast<DiscIO::BlobType>(m_format->currentData().toInt());
 
   m_block_size->clear();
+  m_compression->clear();
+
+  // Populate m_block_size
   switch (format)
   {
   case DiscIO::BlobType::GCZ:
@@ -166,11 +205,90 @@ void ConvertDialog::OnFormatChanged()
 
     break;
   }
+  case DiscIO::BlobType::WIA:
+    m_block_size->setEnabled(true);
+
+    // This is the smallest block size supported by WIA. For performance, larger sizes are avoided.
+    AddToBlockSizeComboBox(0x200000);
+
+    break;
+  case DiscIO::BlobType::RVZ:
+    m_block_size->setEnabled(true);
+
+    for (int block_size = MIN_BLOCK_SIZE; block_size <= MAX_BLOCK_SIZE; block_size *= 2)
+      AddToBlockSizeComboBox(block_size);
+
+    break;
   default:
     break;
   }
 
+  // Populate m_compression
+  switch (format)
+  {
+  case DiscIO::BlobType::GCZ:
+    m_compression->setEnabled(true);
+    AddToCompressionComboBox(QStringLiteral("Deflate"), DiscIO::WIARVZCompressionType::None);
+    break;
+  case DiscIO::BlobType::WIA:
+  case DiscIO::BlobType::RVZ:
+  {
+    m_compression->setEnabled(true);
+
+    // i18n: %1 is the name of a compression method (e.g. LZMA)
+    const QString slow = tr("%1 (slow)");
+
+    AddToCompressionComboBox(tr("No Compression"), DiscIO::WIARVZCompressionType::None);
+
+    if (format == DiscIO::BlobType::WIA)
+      AddToCompressionComboBox(QStringLiteral("Purge"), DiscIO::WIARVZCompressionType::Purge);
+
+    AddToCompressionComboBox(slow.arg(QStringLiteral("bzip2")),
+                             DiscIO::WIARVZCompressionType::Bzip2);
+
+    AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA")), DiscIO::WIARVZCompressionType::LZMA);
+
+    AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA2")),
+                             DiscIO::WIARVZCompressionType::LZMA2);
+
+    if (format == DiscIO::BlobType::RVZ)
+    {
+      AddToCompressionComboBox(QStringLiteral("Zstandard"), DiscIO::WIARVZCompressionType::Zstd);
+      m_compression->setCurrentIndex(m_compression->count() - 1);
+    }
+
+    break;
+  }
+  default:
+    m_compression->setEnabled(false);
+    break;
+  }
+
   m_block_size->setEnabled(m_block_size->count() > 1);
+  m_compression->setEnabled(m_compression->count() > 1);
+
+  m_scrub->setEnabled(format != DiscIO::BlobType::RVZ);
+  if (format == DiscIO::BlobType::RVZ)
+    m_scrub->setChecked(false);
+}
+
+void ConvertDialog::OnCompressionChanged()
+{
+  m_compression_level->clear();
+
+  const auto compression_type =
+      static_cast<DiscIO::WIARVZCompressionType>(m_compression->currentData().toInt());
+
+  const std::pair<int, int> range = DiscIO::GetAllowedCompressionLevels(compression_type);
+
+  for (int i = range.first; i <= range.second; ++i)
+  {
+    AddToCompressionLevelComboBox(i);
+    if (i == 5)
+      m_compression_level->setCurrentIndex(m_compression_level->count() - 1);
+  }
+
+  m_compression_level->setEnabled(m_compression_level->count() > 1);
 }
 
 bool ConvertDialog::ShowAreYouSureDialog(const QString& text)
@@ -189,6 +307,9 @@ void ConvertDialog::Convert()
 {
   const DiscIO::BlobType format = static_cast<DiscIO::BlobType>(m_format->currentData().toInt());
   const int block_size = m_block_size->currentData().toInt();
+  const DiscIO::WIARVZCompressionType compression =
+      static_cast<DiscIO::WIARVZCompressionType>(m_compression->currentData().toInt());
+  const int compression_level = m_compression_level->currentData().toInt();
   const bool scrub = m_scrub->isChecked();
 
   if (scrub && format == DiscIO::BlobType::PLAIN)
@@ -224,7 +345,15 @@ void ConvertDialog::Convert()
     break;
   case DiscIO::BlobType::GCZ:
     extension = QStringLiteral(".gcz");
-    filter = tr("Compressed GC/Wii images (*.gcz)");
+    filter = tr("GCZ GC/Wii images (*.gcz)");
+    break;
+  case DiscIO::BlobType::WIA:
+    extension = QStringLiteral(".wia");
+    filter = tr("WIA GC/Wii images (*.wia)");
+    break;
+  case DiscIO::BlobType::RVZ:
+    extension = QStringLiteral(".rvz");
+    filter = tr("RVZ GC/Wii images (*.rvz)");
     break;
   default:
     ASSERT(false);
@@ -330,8 +459,9 @@ void ConvertDialog::Convert()
     {
       std::future<bool> good;
 
-      if (format == DiscIO::BlobType::PLAIN)
+      switch (format)
       {
+      case DiscIO::BlobType::PLAIN:
         good = std::async(std::launch::async, [&] {
           const bool good =
               DiscIO::ConvertToPlain(blob_reader.get(), original_path, dst_path.toStdString(),
@@ -339,9 +469,9 @@ void ConvertDialog::Convert()
           progress_dialog.Reset();
           return good;
         });
-      }
-      else if (format == DiscIO::BlobType::GCZ)
-      {
+        break;
+
+      case DiscIO::BlobType::GCZ:
         good = std::async(std::launch::async, [&] {
           const bool good =
               DiscIO::ConvertToGCZ(blob_reader.get(), original_path, dst_path.toStdString(),
@@ -350,6 +480,19 @@ void ConvertDialog::Convert()
           progress_dialog.Reset();
           return good;
         });
+        break;
+
+      case DiscIO::BlobType::WIA:
+      case DiscIO::BlobType::RVZ:
+        good = std::async(std::launch::async, [&] {
+          const bool good = DiscIO::ConvertToWIAOrRVZ(
+              blob_reader.get(), original_path, dst_path.toStdString(),
+              format == DiscIO::BlobType::RVZ, compression, compression_level, block_size,
+              &CompressCB, &progress_dialog);
+          progress_dialog.Reset();
+          return good;
+        });
+        break;
       }
 
       progress_dialog.GetRaw()->exec();

Source/Core/DolphinQt/ConvertDialog.h

@@ -14,6 +14,11 @@
 class QCheckBox;
 class QComboBox;
 
+namespace DiscIO
+{
+enum class WIARVZCompressionType : u32;
+}
+
 namespace UICommon
 {
 class GameFile;
@@ -29,15 +34,20 @@ public:
 
 private slots:
   void OnFormatChanged();
+  void OnCompressionChanged();
   void Convert();
 
 private:
   void AddToBlockSizeComboBox(int size);
+  void AddToCompressionComboBox(const QString& name, DiscIO::WIARVZCompressionType type);
+  void AddToCompressionLevelComboBox(int level);
 
   bool ShowAreYouSureDialog(const QString& text);
 
   QComboBox* m_format;
   QComboBox* m_block_size;
+  QComboBox* m_compression;
+  QComboBox* m_compression_level;
   QCheckBox* m_scrub;
   QList<std::shared_ptr<const UICommon::GameFile>> m_files;
 };

Source/Core/DolphinQt/GameList/GameTracker.cpp

@@ -24,6 +24,7 @@ static const QStringList game_filters{
     QStringLiteral("*.[gG][cC][mM]"), QStringLiteral("*.[iI][sS][oO]"),
     QStringLiteral("*.[tT][gG][cC]"), QStringLiteral("*.[cC][iI][sS][oO]"),
     QStringLiteral("*.[gG][cC][zZ]"), QStringLiteral("*.[wW][bB][fF][sS]"),
+    QStringLiteral("*.[wW][iI][aA]"), QStringLiteral("*.[rR][vV][zZ]"),
     QStringLiteral("*.[wW][aA][dD]"), QStringLiteral("*.[eE][lL][fF]"),
     QStringLiteral("*.[dD][oO][lL]")};