RVZ: Store pseudorandom junk data efficiently

Source/Core/DiscIO/CMakeLists.txt

@@ -21,6 +21,8 @@ add_library(discio
   FileSystemGCWii.h
   Filesystem.cpp
   Filesystem.h
+  LaggedFibonacciGenerator.cpp
+  LaggedFibonacciGenerator.h
   MultithreadedCompressor.h
   NANDImporter.cpp
   NANDImporter.h

Source/Core/DiscIO/DiscIO.vcxproj

@@ -55,6 +55,7 @@
     <ClCompile Include="FileBlob.cpp" />
     <ClCompile Include="Filesystem.cpp" />
     <ClCompile Include="FileSystemGCWii.cpp" />
+    <ClCompile Include="LaggedFibonacciGenerator.cpp" />
     <ClCompile Include="NANDImporter.cpp" />
     <ClCompile Include="ScrubbedBlob.cpp" />
     <ClCompile Include="TGCBlob.cpp" />
@@ -81,6 +82,7 @@
     <ClInclude Include="FileBlob.h" />
     <ClInclude Include="Filesystem.h" />
     <ClInclude Include="FileSystemGCWii.h" />
+    <ClInclude Include="LaggedFibonacciGenerator.h" />
     <ClInclude Include="MultithreadedCompressor.h" />
     <ClInclude Include="NANDImporter.h" />
     <ClInclude Include="ScrubbedBlob.h" />

Source/Core/DiscIO/DiscIO.vcxproj.filters

@@ -93,6 +93,9 @@
     <ClCompile Include="WIABlob.cpp">
       <Filter>Volume\Blob</Filter>
     </ClCompile>
+    <ClCompile Include="LaggedFibonacciGenerator.cpp">
+      <Filter>Volume\Blob</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="DiscScrubber.h">
@@ -170,6 +173,9 @@
     <ClInclude Include="WIABlob.h">
       <Filter>Volume\Blob</Filter>
     </ClInclude>
+    <ClInclude Include="LaggedFibonacciGenerator.h">
+      <Filter>Volume\Blob</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Text Include="CMakeLists.txt" />

Source/Core/DiscIO/LaggedFibonacciGenerator.cpp

@@ -0,0 +1,212 @@
+// This file is under the public domain.
+
+#include "DiscIO/LaggedFibonacciGenerator.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstring>
+
+#include "Common/Align.h"
+#include "Common/Assert.h"
+#include "Common/CommonTypes.h"
+#include "Common/Swap.h"
+
+namespace DiscIO
+{
+void LaggedFibonacciGenerator::SetSeed(const u32 seed[SEED_SIZE])
+{
+  SetSeed(reinterpret_cast<const u8*>(seed));
+}
+
+void LaggedFibonacciGenerator::SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)])
+{
+  m_position_bytes = 0;
+
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+    m_buffer[i] = Common::swap32(seed + i * sizeof(u32));
+
+  Initialize(false);
+}
+
+size_t LaggedFibonacciGenerator::GetSeed(const u8* data, size_t size, size_t data_offset,
+                                         u32 seed_out[SEED_SIZE])
+{
+  if ((reinterpret_cast<uintptr_t>(data) - data_offset) % alignof(u32) != 0)
+  {
+    ASSERT(false);
+    return 0;
+  }
+
+  // For code simplicity, only include whole u32 words when regenerating the seed. It would be
+  // possible to get rid of this restriction and use a few additional bytes, but it's probably more
+  // effort than it's worth considering that junk data often starts or ends on 4-byte offsets.
+  const size_t bytes_to_skip = Common::AlignUp(data_offset, sizeof(u32)) - data_offset;
+  const u32* u32_data = reinterpret_cast<const u32*>(data + bytes_to_skip);
+  const size_t u32_size = (size - bytes_to_skip) / sizeof(u32);
+  const size_t u32_data_offset = (data_offset + bytes_to_skip) / sizeof(u32);
+
+  LaggedFibonacciGenerator lfg;
+  if (!GetSeed(u32_data, u32_size, u32_data_offset, &lfg, seed_out))
+    return false;
+
+  lfg.m_position_bytes = data_offset % (LFG_K * sizeof(u32));
+
+  const u8* end = data + size;
+  size_t reconstructed_bytes = 0;
+  while (data < end && lfg.GetByte() == *data)
+  {
+    ++reconstructed_bytes;
+    ++data;
+  }
+  return reconstructed_bytes;
+}
+
+bool LaggedFibonacciGenerator::GetSeed(const u32* data, size_t size, size_t data_offset,
+                                       LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE])
+{
+  if (size < LFG_K)
+    return false;
+
+  // If the data doesn't look like something we can regenerate, return early to save time
+  if (!std::all_of(data, data + LFG_K, [](u32 x) {
+        return (Common::swap32(x) & 0x00C00000) == (Common::swap32(x) >> 2 & 0x00C00000);
+      }))
+  {
+    return false;
+  }
+
+  const size_t data_offset_mod_k = data_offset % LFG_K;
+  const size_t data_offset_div_k = data_offset / LFG_K;
+
+  std::copy(data, data + LFG_K - data_offset_mod_k, lfg->m_buffer.data() + data_offset_mod_k);
+  std::copy(data + LFG_K - data_offset_mod_k, data + LFG_K, lfg->m_buffer.data());
+
+  lfg->Backward(0, data_offset_mod_k);
+
+  for (size_t i = 0; i < data_offset_div_k; ++i)
+    lfg->Backward();
+
+  if (!lfg->Reinitialize(seed_out))
+    return false;
+
+  for (size_t i = 0; i < data_offset_div_k; ++i)
+    lfg->Forward();
+
+  return true;
+}
+
+void LaggedFibonacciGenerator::GetBytes(size_t count, u8* out)
+{
+  while (count > 0)
+  {
+    const size_t length = std::min(count, LFG_K * sizeof(u32) - m_position_bytes);
+
+    std::memcpy(out, reinterpret_cast<u8*>(m_buffer.data()) + m_position_bytes, length);
+
+    m_position_bytes += length;
+    count -= length;
+    out += length;
+
+    if (m_position_bytes == LFG_K * sizeof(u32))
+    {
+      Forward();
+      m_position_bytes = 0;
+    }
+  }
+}
+
+u8 LaggedFibonacciGenerator::GetByte()
+{
+  const u8 result = reinterpret_cast<u8*>(m_buffer.data())[m_position_bytes];
+
+  ++m_position_bytes;
+
+  if (m_position_bytes == LFG_K * sizeof(u32))
+  {
+    Forward();
+    m_position_bytes = 0;
+  }
+
+  return result;
+}
+
+void LaggedFibonacciGenerator::Forward(size_t count)
+{
+  m_position_bytes += count;
+  while (m_position_bytes >= LFG_K * sizeof(u32))
+  {
+    Forward();
+    m_position_bytes -= LFG_K * sizeof(u32);
+  }
+}
+
+void LaggedFibonacciGenerator::Forward()
+{
+  for (size_t i = 0; i < LFG_J; ++i)
+    m_buffer[i] ^= m_buffer[i + LFG_K - LFG_J];
+
+  for (size_t i = LFG_J; i < LFG_K; ++i)
+    m_buffer[i] ^= m_buffer[i - LFG_J];
+}
+
+void LaggedFibonacciGenerator::Backward(size_t start_word, size_t end_word)
+{
+  const size_t loop_end = std::max(LFG_J, start_word);
+  for (size_t i = std::min(end_word, LFG_K); i > loop_end; --i)
+    m_buffer[i - 1] ^= m_buffer[i - 1 - LFG_J];
+
+  for (size_t i = std::min(end_word, LFG_J); i > start_word; --i)
+    m_buffer[i - 1] ^= m_buffer[i - 1 + LFG_K - LFG_J];
+}
+
+bool LaggedFibonacciGenerator::Reinitialize(u32 seed_out[SEED_SIZE])
+{
+  for (size_t i = 0; i < 4; ++i)
+    Backward();
+
+  for (u32& x : m_buffer)
+    x = Common::swap32(x);
+
+  // Reconstruct the bits which are missing due to the output code shifting by 18 instead of 16.
+  // Unfortunately we can't reconstruct bits 16 and 17 (counting LSB as 0) for the first word,
+  // but the observable result (when shifting by 18 instead of 16) is not affected by this.
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+  {
+    m_buffer[i] = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000) |
+                  ((m_buffer[i + 16] ^ m_buffer[i + 15]) << 9 & 0x00030000);
+  }
+
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+    seed_out[i] = Common::swap32(m_buffer[i]);
+
+  return Initialize(true);
+}
+
+bool LaggedFibonacciGenerator::Initialize(bool check_existing_data)
+{
+  for (size_t i = SEED_SIZE; i < LFG_K; ++i)
+  {
+    const u32 calculated = (m_buffer[i - 17] << 23) ^ (m_buffer[i - 16] >> 9) ^ m_buffer[i - 1];
+
+    if (check_existing_data)
+    {
+      const u32 actual = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000);
+      if ((calculated & 0xFFFCFFFF) != actual)
+        return false;
+    }
+
+    m_buffer[i] = calculated;
+  }
+
+  // Instead of doing the "shift by 18 instead of 16" oddity when actually outputting the data,
+  // we can do the shifting (and byteswapping) at this point to make the output code simpler.
+  for (u32& x : m_buffer)
+    x = Common::swap32((x & 0xFF00FFFF) | ((x >> 2) & 0x00FF0000));
+
+  for (size_t i = 0; i < 4; ++i)
+    Forward();
+
+  return true;
+}
+
+}  // namespace DiscIO

Source/Core/DiscIO/LaggedFibonacciGenerator.h

@@ -0,0 +1,51 @@
+// This file is under the public domain.
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+
+#include "Common/CommonTypes.h"
+
+namespace DiscIO
+{
+class LaggedFibonacciGenerator
+{
+public:
+  static constexpr size_t SEED_SIZE = 17;
+
+  // Reconstructs a seed and writes it to seed_out, then returns the number of bytes which can
+  // be reconstructed using that seed. Can return any number between 0 and size, inclusive.
+  // data - data_offset must be 4-byte aligned.
+  static size_t GetSeed(const u8* data, size_t size, size_t data_offset, u32 seed_out[SEED_SIZE]);
+
+  // SetSeed must be called before using the functions below
+  void SetSeed(const u32 seed[SEED_SIZE]);
+  void SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)]);
+
+  // Outputs a number of bytes and advances the internal state by the same amount.
+  void GetBytes(size_t count, u8* out);
+  u8 GetByte();
+
+  // Advances the internal state like GetBytes, but without outputting data. O(N), like GetBytes.
+  void Forward(size_t count);
+
+private:
+  static bool GetSeed(const u32* data, size_t size, size_t data_offset,
+                      LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE]);
+
+  void Forward();
+  void Backward(size_t start_word = 0, size_t end_word = LFG_K);
+
+  bool Reinitialize(u32 seed_out[SEED_SIZE]);
+  bool Initialize(bool check_existing_data);
+
+  static constexpr size_t LFG_K = 521;
+  static constexpr size_t LFG_J = 32;
+
+  std::array<u32, LFG_K> m_buffer;
+
+  size_t m_position_bytes = 0;
+};
+
+}  // namespace DiscIO

Source/Core/DiscIO/WIABlob.h

@@ -10,6 +10,7 @@
 #include <memory>
 #include <mutex>
 #include <optional>
+#include <type_traits>
 #include <utility>
 
 #include <bzlib.h>
@@ -21,6 +22,7 @@
 #include "Common/File.h"
 #include "Common/Swap.h"
 #include "DiscIO/Blob.h"
+#include "DiscIO/LaggedFibonacciGenerator.h"
 #include "DiscIO/MultithreadedCompressor.h"
 #include "DiscIO/WiiEncryptionCache.h"
 
@@ -267,6 +269,31 @@ private:
     ZSTD_DStream* m_stream;
   };
 
+  class RVZPackDecompressor final : public Decompressor
+  {
+  public:
+    RVZPackDecompressor(std::unique_ptr<Decompressor> decompressor,
+                        DecompressionBuffer decompressed, u64 data_offset);
+
+    bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                    size_t* in_bytes_read) override;
+
+    bool Done() const override;
+
+  private:
+    std::optional<bool> ReadToDecompressed(const DecompressionBuffer& in, size_t* in_bytes_read,
+                                           size_t decompressed_bytes_read, size_t bytes_to_read);
+
+    std::unique_ptr<Decompressor> m_decompressor;
+    DecompressionBuffer m_decompressed;
+    size_t m_decompressed_bytes_read = 0;
+    u64 m_data_offset;
+
+    u32 m_size = 0;
+    bool m_junk;
+    LaggedFibonacciGenerator m_lfg;
+  };
+
   class Compressor
   {
   public:
@@ -375,7 +402,7 @@ private:
   public:
     Chunk();
     Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size,
-          u32 exception_lists, bool compressed_exception_lists,
+          u32 exception_lists, bool compressed_exception_lists, bool rvz_pack, u64 data_offset,
           std::unique_ptr<Decompressor> decompressor);
 
     bool Read(u64 offset, u64 size, u8* out_ptr);
@@ -391,6 +418,7 @@ private:
     }
 
   private:
+    bool Decompress();
     bool HandleExceptions(const u8* data, size_t bytes_allocated, size_t bytes_written,
                           size_t* bytes_used, bool align);
 
@@ -407,6 +435,8 @@ private:
     size_t m_in_bytes_used_for_exceptions = 0;
     u32 m_exception_lists = 0;
     bool m_compressed_exception_lists = false;
+    bool m_rvz_pack = false;
+    u64 m_data_offset = 0;
   };
 
   explicit WIAFileReader(File::IOFile file, const std::string& path);
@@ -417,7 +447,7 @@ private:
                       u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups,
                       u32 exception_lists);
   Chunk& ReadCompressedData(u64 offset_in_file, u64 compressed_size, u64 decompressed_size,
-                            u32 exception_lists);
+                            u32 exception_lists = 0, bool rvz_pack = false, u64 data_offset = 0);
 
   static bool ApplyHashExceptions(const std::vector<HashExceptionEntry>& exception_list,
                                   VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]);
@@ -430,18 +460,18 @@ private:
   {
     bool operator==(const ReuseID& other) const
     {
-      return std::tie(partition_key, data_size, decrypted, value) ==
-             std::tie(other.partition_key, other.data_size, other.decrypted, other.value);
+      return std::tie(partition_key, data_size, encrypted, value) ==
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
     }
     bool operator<(const ReuseID& other) const
     {
-      return std::tie(partition_key, data_size, decrypted, value) <
-             std::tie(other.partition_key, other.data_size, other.decrypted, other.value);
+      return std::tie(partition_key, data_size, encrypted, value) <
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
     }
     bool operator>(const ReuseID& other) const
     {
-      return std::tie(partition_key, data_size, decrypted, value) >
-             std::tie(other.partition_key, other.data_size, other.decrypted, other.value);
+      return std::tie(partition_key, data_size, encrypted, value) >
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
     }
     bool operator!=(const ReuseID& other) const { return !operator==(other); }
     bool operator>=(const ReuseID& other) const { return !operator<(other); }
@@ -449,7 +479,7 @@ private:
 
     const WiiKey* partition_key;
     u64 data_size;
-    bool decrypted;
+    bool encrypted;
     u8 value;
   };
 
@@ -470,6 +500,7 @@ private:
   {
     std::vector<u8> data;
     const DataEntry* data_entry;
+    u64 data_offset;
     u64 bytes_read;
     size_t group_index;
   };
@@ -512,13 +543,17 @@ private:
                               WIAHeader2* header_2);
   static bool TryReuse(std::map<ReuseID, GroupEntry>* reusable_groups,
                        std::mutex* reusable_groups_mutex, OutputParametersEntry* entry);
+  static void RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chunk, size_t chunks,
+                      u64 total_size, u64 data_offset, u64 in_offset, bool allow_junk_reuse);
+  static void RVZPack(const u8* in, OutputParametersEntry* out, u64 size, u64 data_offset,
+                      bool allow_junk_reuse);
   static ConversionResult<OutputParameters>
   ProcessAndCompress(CompressThreadState* state, CompressParameters parameters,
                      const std::vector<PartitionEntry>& partition_entries,
                      const std::vector<DataEntry>& data_entries,
                      std::map<ReuseID, GroupEntry>* reusable_groups,
                      std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group,
-                     u64 exception_lists_per_chunk, bool compressed_exception_lists);
+                     u64 exception_lists_per_chunk, bool compressed_exception_lists, bool rvz);
   static ConversionResultCode Output(std::vector<OutputParametersEntry>* entries,
                                      File::IOFile* outfile,
                                      std::map<ReuseID, GroupEntry>* reusable_groups,
@@ -528,13 +563,20 @@ private:
                                           u32 total_groups, u64 iso_size, CompressCB callback,
                                           void* arg);
 
+  static void PushBack(std::vector<u8>* vector, const u8* begin, const u8* end)
+  {
+    const size_t offset_in_vector = vector->size();
+    vector->resize(offset_in_vector + (end - begin));
+    std::copy(begin, end, vector->data() + offset_in_vector);
+  }
+
   template <typename T>
   static void PushBack(std::vector<u8>* vector, const T& x)
   {
-    const size_t offset_in_vector = vector->size();
-    vector->resize(offset_in_vector + sizeof(T));
+    static_assert(std::is_trivially_copyable_v<T>);
+
     const u8* x_ptr = reinterpret_cast<const u8*>(&x);
-    std::copy(x_ptr, x_ptr + sizeof(T), vector->data() + offset_in_vector);
+    PushBack(vector, x_ptr, x_ptr + sizeof(T));
   }
 
   bool m_valid;
@@ -566,9 +608,9 @@ private:
   static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000;
   static constexpr u32 WIA_VERSION_READ_COMPATIBLE = 0x00080000;
 
-  static constexpr u32 RVZ_VERSION = 0x00010000;
-  static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00010000;
-  static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00010000;
+  static constexpr u32 RVZ_VERSION = 0x00020000;
+  static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00020000;
+  static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00020000;
 };
 
 }  // namespace DiscIO

Source/Core/DolphinQt/ConvertDialog.cpp

@@ -252,6 +252,10 @@ void ConvertDialog::OnFormatChanged()
 
   m_block_size->setEnabled(m_block_size->count() > 1);
   m_compression->setEnabled(m_compression->count() > 1);
+
+  m_scrub->setEnabled(format != DiscIO::BlobType::RVZ);
+  if (format == DiscIO::BlobType::RVZ)
+    m_scrub->setChecked(false);
 }
 
 void ConvertDialog::OnCompressionChanged()