diff --git a/Cargo.lock b/Cargo.lock index 8b2851b45..b641b840c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -636,6 +636,35 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc-fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61dd931a9d071dc6e36b8735c501601428c5aa5dfd6d2506b935498ef57e0098" +dependencies = [ + "crc", + "digest", + "libc", + "rand 0.9.2", + "regex", + "rustversion", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -4081,6 +4110,7 @@ dependencies = [ "bstr", "chrono", "clap", + "crc-fast", "crc32fast", "data-encoding", "data-encoding-macro", diff --git a/Cargo.toml b/Cargo.toml index b76faebf9..0fbe7ee1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -391,6 +391,7 @@ blake2b_simd = "1.0.2" blake3 = "1.5.1" sm3 = "0.4.2" crc32fast = "1.4.2" +crc-fast = "1.5.0" digest = "0.10.7" # Fluent dependencies diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 414cf43fd..1f0c111c6 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -323,6 +323,35 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc-fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61dd931a9d071dc6e36b8735c501601428c5aa5dfd6d2506b935498ef57e0098" +dependencies = [ + "crc", + "digest", + "libc", + "rand", + "regex", + "rustversion", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -1618,6 +1647,7 @@ dependencies = [ "blake3", "bstr", "clap", + "crc-fast", "crc32fast", "data-encoding", "data-encoding-macro", diff --git a/src/uucore/Cargo.toml b/src/uucore/Cargo.toml index b34543372..720721c63 100644 --- a/src/uucore/Cargo.toml +++ b/src/uucore/Cargo.toml @@ -58,6 +58,7 @@ blake2b_simd = { workspace = true, optional = true } blake3 = { workspace = true, optional = true } sm3 = { workspace = true, optional = true } crc32fast = { workspace = true, optional = true } +crc-fast = { workspace = true, optional = true } bigdecimal = { workspace = true, optional = true } num-traits = { workspace = true, optional = true } selinux = { workspace = true, optional = true } @@ -152,6 +153,7 @@ sum = [ "blake3", "sm3", "crc32fast", + "crc-fast", ] update-control = ["parser"] utf8 = [] diff --git a/src/uucore/src/lib/features/sum.rs b/src/uucore/src/lib/features/sum.rs index fce0fd89e..5ede05e8f 100644 --- a/src/uucore/src/lib/features/sum.rs +++ b/src/uucore/src/lib/features/sum.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore memmem algo +// spell-checker:ignore memmem algo PCLMULQDQ refin xorout //! Implementations of digest functions, like md5 and sha1. //! @@ -18,6 +18,22 @@ use hex::encode; #[cfg(windows)] use memchr::memmem; +use crc_fast::{CrcParams, checksum_with_params}; + +// POSIX cksum SIMD configuration for crc-fast +// This uses SIMD instructions (PCLMULQDQ) for fast CRC computation +fn get_posix_cksum_params() -> CrcParams { + CrcParams::new( + "CRC-32/CKSUM", // Name + 32, // Width + 0x04c11db7, // Polynomial + 0x00000000, // Initial CRC value: 0 (not 0xffffffff) + false, // No input reflection (refin) + 0xffffffff, // XOR output with 0xffffffff (xorout) + 0, // Check value (not used) + ) +} + pub trait Digest { fn new() -> Self where @@ -122,48 +138,11 @@ impl Digest for Sm3 { } } -// NOTE: CRC_TABLE_LEN *must* be <= 256 as we cast 0..CRC_TABLE_LEN to u8 -const CRC_TABLE_LEN: usize = 256; - pub struct Crc { state: u32, size: usize, - crc_table: [u32; CRC_TABLE_LEN], -} -impl Crc { - fn generate_crc_table() -> [u32; CRC_TABLE_LEN] { - let mut table = [0; CRC_TABLE_LEN]; - - for (i, elt) in table.iter_mut().enumerate().take(CRC_TABLE_LEN) { - *elt = Self::crc_entry(i as u8); - } - - table - } - fn crc_entry(input: u8) -> u32 { - let mut crc = (input as u32) << 24; - - let mut i = 0; - while i < 8 { - let if_condition = crc & 0x8000_0000; - let if_body = (crc << 1) ^ 0x04c1_1db7; - let else_body = crc << 1; - - // NOTE: i feel like this is easier to understand than emulating an if statement in bitwise - // ops - let condition_table = [else_body, if_body]; - - crc = condition_table[(if_condition != 0) as usize]; - i += 1; - } - - crc - } - - fn update(&mut self, input: u8) { - self.state = (self.state << 8) - ^ self.crc_table[((self.state >> 24) as usize ^ input as usize) & 0xFF]; - } + // Store data for SIMD processing + data_buffer: Vec, } impl Digest for Crc { @@ -171,24 +150,26 @@ impl Digest for Crc { Self { state: 0, size: 0, - crc_table: Self::generate_crc_table(), + data_buffer: Vec::with_capacity(8192), } } fn hash_update(&mut self, input: &[u8]) { - for &elt in input { - self.update(elt); - } self.size += input.len(); + // Store data for SIMD processing + self.data_buffer.extend_from_slice(input); } fn hash_finalize(&mut self, out: &mut [u8]) { + // Add the size bytes to the data buffer let mut sz = self.size; while sz != 0 { - self.update(sz as u8); + self.data_buffer.push(sz as u8); sz >>= 8; } - self.state = !self.state; + + // Use SIMD-accelerated CRC computation + self.state = checksum_with_params(get_posix_cksum_params(), &self.data_buffer) as u32; out.copy_from_slice(&self.state.to_ne_bytes()); } @@ -199,7 +180,9 @@ impl Digest for Crc { } fn reset(&mut self) { - *self = Self::new(); + self.state = 0; + self.size = 0; + self.data_buffer.clear(); } fn output_bits(&self) -> usize { @@ -529,4 +512,133 @@ mod tests { assert_eq!(result_crlf, result_lf); } + + use super::{Crc, Digest}; + + #[test] + fn test_crc_basic_functionality() { + // Test that our CRC implementation works with basic functionality + let mut crc1 = Crc::new(); + let mut crc2 = Crc::new(); + + // Same input should give same output + crc1.hash_update(b"test"); + crc2.hash_update(b"test"); + + let mut out1 = [0u8; 4]; + let mut out2 = [0u8; 4]; + crc1.hash_finalize(&mut out1); + crc2.hash_finalize(&mut out2); + + assert_eq!(out1, out2); + } + + #[test] + fn test_crc_digest_basic() { + let mut crc = Crc::new(); + + // Test empty input + let mut output = [0u8; 4]; + crc.hash_finalize(&mut output); + let empty_result = u32::from_ne_bytes(output); + + // Reset and test with "test" string + crc.reset(); + crc.hash_update(b"test"); + crc.hash_finalize(&mut output); + let test_result = u32::from_ne_bytes(output); + + // The result should be different for different inputs + assert_ne!(empty_result, test_result); + + // Test known value: "test" should give 3076352578 + assert_eq!(test_result, 3076352578); + } + + #[test] + fn test_crc_digest_incremental() { + let mut crc1 = Crc::new(); + let mut crc2 = Crc::new(); + + // Test that processing in chunks gives same result as all at once + let data = b"Hello, World! This is a test string for CRC computation."; + + // Process all at once + crc1.hash_update(data); + let mut output1 = [0u8; 4]; + crc1.hash_finalize(&mut output1); + + // Process in chunks + crc2.hash_update(&data[0..10]); + crc2.hash_update(&data[10..30]); + crc2.hash_update(&data[30..]); + let mut output2 = [0u8; 4]; + crc2.hash_finalize(&mut output2); + + assert_eq!(output1, output2); + } + + #[test] + fn test_crc_slice8_vs_single_byte() { + // Test that our optimized slice-by-8 gives same results as byte-by-byte + let test_data = b"This is a longer test string to verify slice-by-8 optimization works correctly with various data sizes including remainders."; + + let mut crc_optimized = Crc::new(); + crc_optimized.hash_update(test_data); + let mut output_opt = [0u8; 4]; + crc_optimized.hash_finalize(&mut output_opt); + + // Create a reference implementation using hash_update + let mut crc_reference = Crc::new(); + for &byte in test_data { + crc_reference.hash_update(&[byte]); + } + let mut output_ref = [0u8; 4]; + crc_reference.hash_finalize(&mut output_ref); + + assert_eq!(output_opt, output_ref); + } + + #[test] + fn test_crc_known_values() { + // Test against our CRC implementation values + // Note: These are the correct values for our POSIX cksum implementation + let test_cases = [ + ("", 4294967295u32), + ("a", 1220704766u32), + ("abc", 1219131554u32), + ]; + + for (input, expected) in test_cases { + let mut crc = Crc::new(); + crc.hash_update(input.as_bytes()); + let mut output = [0u8; 4]; + crc.hash_finalize(&mut output); + let result = u32::from_ne_bytes(output); + + assert_eq!(result, expected, "CRC mismatch for input: '{}'", input); + } + } + + #[test] + fn test_crc_hash_update_edge_cases() { + let mut crc = Crc::new(); + + // Test with data that's not a multiple of 8 bytes + let data7 = b"1234567"; // 7 bytes + crc.hash_update(data7); + + let data9 = b"123456789"; // 9 bytes + let mut crc2 = Crc::new(); + crc2.hash_update(data9); + + // Should not panic and should produce valid results + let mut out1 = [0u8; 4]; + let mut out2 = [0u8; 4]; + crc.hash_finalize(&mut out1); + crc2.hash_finalize(&mut out2); + + // Results should be different for different inputs + assert_ne!(out1, out2); + } }