Simple WBFS partition stripping (#4)

* Strip partitions from WBFS

* More concise code

* Draft: add ProcessOptions::scrub_update_partition

* add scrub_update_partition for CISO

* update comment

* fix doctests

* #[allow(clippy::too_many_arguments)]

* ⬆️ Update deps

* ⬆️ Update deps

* 🚑 Fix compilation

* run cargo update

* run cargo fmt

* Revert "run cargo fmt"

This reverts commit 0f1360779c.

* Revert "run cargo update"

This reverts commit caa13433a2.

* Add ScrubLevel enum & fix formatting

* Update usages & fix block-padding

---------

Co-authored-by: Luke Street <luke@street.dev>

Cargo.toml

@@ -21,8 +21,8 @@ repository = "https://github.com/encounter/nod"
 keywords = ["gamecube", "wii", "iso", "wbfs", "rvz"]
 
 [workspace.dependencies]
-digest = { version = "0.11.0-rc.0", default-features = false }
-md-5 = { version = "0.11.0-rc.0", default-features = false }
-sha1 = { version = "0.11.0-rc.0", default-features = false }
+digest = { version = "0.11.0-rc.3", default-features = false }
+md-5 = { version = "0.11.0-rc.2", default-features = false }
+sha1 = { version = "0.11.0-rc.2", default-features = false }
 tracing = "0.1"
 zerocopy = { version = "0.8", features = ["alloc", "derive"] }

nod/Cargo.toml

@@ -25,11 +25,12 @@ openssl-vendored = ["openssl", "openssl/vendored"]
 
 [dependencies]
 adler2 = { version = "2.0", optional = true }
-aes = "0.9.0-rc.0"
-base16ct = "0.2"
+aes = "0.9.0-rc.1"
+base16ct = "0.3"
+block-padding = "=0.4.0-rc.4"
 bytes = "1.10"
 bzip2 = { version = "0.6", features = ["static"], optional = true }
-cbc = "0.2.0-rc.0"
+cbc = "0.2.0-rc.1"
 crc32fast = "1.5"
 crossbeam-channel = "0.5"
 crossbeam-utils = "0.8"
@@ -52,4 +53,3 @@ xxhash-rust = { version = "0.8", features = ["xxh64"] }
 zerocopy = { workspace = true }
 zstd = { version = "0.13", optional = true, default-features = false }
 zstd-safe = { version = "7.2", optional = true, default-features = false }
-

nod/src/disc/writer.rs

@@ -9,7 +9,7 @@ use dyn_clone::DynClone;
 
 use crate::{
     Error, Result, ResultContext,
-    common::PartitionInfo,
+    common::{PartitionInfo, PartitionKind},
     disc::{
         SECTOR_SIZE,
         reader::DiscReader,
@@ -192,6 +192,7 @@ pub enum CheckBlockResult {
 }
 
 /// Check if a block is zeroed or junk data.
+#[allow(clippy::too_many_arguments)]
 pub(crate) fn check_block(
     buf: &[u8],
     decrypted_block: &mut [u8],
@@ -200,12 +201,18 @@ pub(crate) fn check_block(
     lfg: &mut LaggedFibonacci,
     disc_id: [u8; 4],
     disc_num: u8,
+    scrub_update_partition: bool,
 ) -> io::Result<CheckBlockResult> {
     let start_sector = (input_position / SECTOR_SIZE as u64) as u32;
     let end_sector = ((input_position + buf.len() as u64) / SECTOR_SIZE as u64) as u32;
     if let Some(partition) = partition_info.iter().find(|p| {
         p.has_hashes && start_sector >= p.data_start_sector && end_sector < p.data_end_sector
     }) {
+        // Ignore update partition data
+        if scrub_update_partition && partition.kind == PartitionKind::Update {
+            return Ok(CheckBlockResult::Zeroed);
+        }
+
         if input_position % SECTOR_SIZE as u64 != 0 {
             return Err(io::Error::other("Partition block not aligned to sector boundary"));
         }

nod/src/io/ciso.rs

@@ -31,7 +31,9 @@ use crate::{
         read::{box_to_bytes, read_arc_at},
         static_assert,
     },
-    write::{DataCallback, DiscFinalization, DiscWriterWeight, FormatOptions, ProcessOptions},
+    write::{
+        DataCallback, DiscFinalization, DiscWriterWeight, FormatOptions, ProcessOptions, ScrubLevel,
+    },
 };
 
 pub const CISO_MAP_SIZE: usize = SECTOR_SIZE - 8;
@@ -145,6 +147,7 @@ struct BlockProcessorCISO {
     lfg: LaggedFibonacci,
     disc_id: [u8; 4],
     disc_num: u8,
+    scrub_update_partition: bool,
 }
 
 impl Clone for BlockProcessorCISO {
@@ -156,6 +159,7 @@ impl Clone for BlockProcessorCISO {
             lfg: LaggedFibonacci::default(),
             disc_id: self.disc_id,
             disc_num: self.disc_num,
+            scrub_update_partition: self.scrub_update_partition,
         }
     }
 }
@@ -178,6 +182,7 @@ impl BlockProcessor for BlockProcessorCISO {
             &mut self.lfg,
             self.disc_id,
             self.disc_num,
+            self.scrub_update_partition,
         )? {
             CheckBlockResult::Normal => {
                 BlockResult { block_idx, disc_data, block_data, meta: CheckBlockResult::Normal }
@@ -264,6 +269,7 @@ impl DiscWriter for DiscWriterCISO {
                 lfg: LaggedFibonacci::default(),
                 disc_id,
                 disc_num,
+                scrub_update_partition: options.scrub == ScrubLevel::UpdatePartition,
             },
             self.block_count,
             options.processor_threads,

nod/src/io/wbfs.rs

@@ -30,7 +30,9 @@ use crate::{
         lfg::LaggedFibonacci,
         read::{read_arc_slice_at, read_at, read_box_slice_at},
     },
-    write::{DataCallback, DiscFinalization, DiscWriterWeight, FormatOptions, ProcessOptions},
+    write::{
+        DataCallback, DiscFinalization, DiscWriterWeight, FormatOptions, ProcessOptions, ScrubLevel,
+    },
 };
 
 #[derive(Debug, Clone, PartialEq, FromBytes, IntoBytes, Immutable, KnownLayout)]
@@ -165,6 +167,7 @@ struct BlockProcessorWBFS {
     lfg: LaggedFibonacci,
     disc_id: [u8; 4],
     disc_num: u8,
+    scrub_update_partition: bool,
 }
 
 impl Clone for BlockProcessorWBFS {
@@ -177,6 +180,7 @@ impl Clone for BlockProcessorWBFS {
             lfg: LaggedFibonacci::default(),
             disc_id: self.disc_id,
             disc_num: self.disc_num,
+            scrub_update_partition: self.scrub_update_partition,
         }
     }
 }
@@ -199,6 +203,7 @@ impl BlockProcessor for BlockProcessorWBFS {
             &mut self.lfg,
             self.disc_id,
             self.disc_num,
+            self.scrub_update_partition,
         )? {
             CheckBlockResult::Normal => {
                 BlockResult { block_idx, disc_data, block_data, meta: CheckBlockResult::Normal }
@@ -235,6 +240,7 @@ impl DiscWriterWBFS {
         if options.format != Format::Wbfs {
             return Err(Error::DiscFormat("Invalid format for WBFS writer".to_string()));
         }
+
         if options.compression != Compression::None {
             return Err(Error::DiscFormat("WBFS does not support compression".to_string()));
         }
@@ -317,6 +323,7 @@ impl DiscWriter for DiscWriterWBFS {
                 lfg: LaggedFibonacci::default(),
                 disc_id,
                 disc_num,
+                scrub_update_partition: options.scrub == ScrubLevel::UpdatePartition,
             },
             self.block_count as u32,
             options.processor_threads,

nod/src/lib.rs

@@ -83,7 +83,7 @@
 //! use nod::{
 //!     common::{Compression, Format},
 //!     read::{DiscOptions, DiscReader, PartitionEncryption},
-//!     write::{DiscWriter, DiscWriterWeight, FormatOptions, ProcessOptions},
+//!     write::{DiscWriter, DiscWriterWeight, FormatOptions, ProcessOptions, ScrubLevel},
 //! };
 //!
 //! let open_options = DiscOptions {
@@ -121,6 +121,7 @@
 //!     digest_md5: false, // MD5 is slow, skip it
 //!     digest_sha1: true,
 //!     digest_xxh64: true,
+//!     scrub: ScrubLevel::None,
 //! };
 //! // Start processing the disc image.
 //! let finalization = writer

nod/src/write.rs

@@ -70,6 +70,25 @@ pub struct ProcessOptions {
     /// Each digest calculation will run on a separate thread, unaffected by the processor thread
     /// count.
     pub digest_xxh64: bool,
+    /// The level of scrubbing to perform on the disc image.
+    ///
+    /// This may reduce the size of the output disc image by removing unnecessary data, but will
+    /// also prevent reconstruction of the original disc image. Use with caution.
+    ///
+    /// If unsure, use `ScrubLevel::None`.
+    pub scrub: ScrubLevel,
+}
+
+/// The level of scrubbing to perform on the disc image.
+#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ScrubLevel {
+    /// Do not scrub any data from the disc image.
+    #[default]
+    None,
+    /// Replace the update partition with zeroes to save space.
+    ///
+    /// NOTE: This is currently implemented only for WBFS and CISO.
+    UpdatePartition,
 }
 
 /// A callback for writing disc data.

nodtool/Cargo.toml

@@ -29,7 +29,7 @@ tracy = ["dep:tracing-tracy"]
 argp = "0.4"
 crc32fast = "1.5"
 digest = { workspace = true }
-enable-ansi-support = "0.2"
+enable-ansi-support = "0.3"
 hex = { version = "0.4", features = ["serde"] }
 indicatif = "0.18"
 md-5 = { workspace = true }

nodtool/src/cmd/dat.rs

@@ -9,7 +9,7 @@ use indicatif::{ProgressBar, ProgressState, ProgressStyle};
 use nod::{
     Result, ResultContext,
     read::{DiscOptions, DiscReader, PartitionEncryption},
-    write::{DiscWriter, FormatOptions, ProcessOptions},
+    write::{DiscWriter, FormatOptions, ProcessOptions, ScrubLevel},
 };
 
 use crate::util::{redump, redump::GameResult};
@@ -192,6 +192,7 @@ fn load_disc(path: &Path, name: &str, full_verify: bool) -> Result<DiscHashes> {
             digest_md5: false,
             digest_sha1: true,
             digest_xxh64: false,
+            scrub: ScrubLevel::None,
         },
     )?;
     pb.finish();

nodtool/src/util/shared.rs

@@ -11,7 +11,7 @@ use nod::{
     common::Compression,
     disc::DiscHeader,
     read::{DiscMeta, DiscOptions, DiscReader, PartitionEncryption},
-    write::{DiscWriter, DiscWriterWeight, FormatOptions, ProcessOptions},
+    write::{DiscWriter, DiscWriterWeight, FormatOptions, ProcessOptions, ScrubLevel},
 };
 use size::Size;
 
@@ -123,6 +123,7 @@ pub fn convert_and_verify(
             digest_md5: md5,
             digest_sha1: true,
             digest_xxh64: true,
+            scrub: ScrubLevel::None,
         },
     )?;
     pb.finish();