Speed up CRC32 calculation

It turns out that passing in one byte at a time is really inefficient
(which really should have been obvious...). For small files the
difference is insignificant, but for Skyrim SE's Update.esm (18 MB)
feeding in the buffered file content slice made the calculation twice
as fast, and this seems to have non-linear effect, as the benchmark
against Dragonborn.esm (63 MB) would have taken ~ 1137 seconds to run
when passing one byte at a time (the benchmark using a buffer took ~ 5
seconds).

On top of that, switching to the crc32fast crate has an insignificant
effect on benchmarking with Blank.esm, but Update.esm's benchmark
was twice as fast again, and when benchmarking with Dragonborn.esm,
using crc32fast was ~ 170x faster than using the crc crate!
This commit is contained in:
Oliver Hamlet
2020-06-13 23:17:32 +01:00
parent 3b1e54a4ed
commit d0bde327c2
2 changed files with 14 additions and 10 deletions
+1 -1
View File
@@ -6,7 +6,7 @@ license = "MIT"
edition = "2018"
[dependencies]
crc = "1.0.0"
crc32fast = "1.2.0"
esplugin = "3.0.0"
nom = "5.0.0"
pelite = "0.8.0"
+13 -9
View File
@@ -1,10 +1,9 @@
use std::ffi::OsStr;
use std::fs::{read_dir, File};
use std::hash::Hasher;
use std::io::{BufReader, Read};
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};
use crc::{crc32, Hasher32};
use regex::Regex;
use super::{ComparisonOperator, Function};
@@ -157,16 +156,21 @@ fn evaluate_checksum(state: &State, file_path: &Path, crc: u32) -> Result<bool,
return Ok(false);
}
let file = File::open(path).map_err(|e| Error::IoError(file_path.to_path_buf(), e))?;
let reader = BufReader::new(file);
let mut digest = crc32::Digest::new(crc32::IEEE);
let io_error_mapper = |e| Error::IoError(file_path.to_path_buf(), e);
let file = File::open(path).map_err(io_error_mapper)?;
let mut reader = BufReader::new(file);
let mut hasher = crc32fast::Hasher::new();
for byte in reader.bytes() {
let byte = byte.map_err(|e| Error::IoError(file_path.to_path_buf(), e))?;
digest.write_u8(byte);
let mut buffer = reader.fill_buf().map_err(io_error_mapper)?;
while !buffer.is_empty() {
hasher.write(buffer);
let length = buffer.len();
reader.consume(length);
buffer = reader.fill_buf().map_err(io_error_mapper)?;
}
let calculated_crc = digest.sum32();
let calculated_crc = hasher.finalize();
if let Ok(mut writer) = state.crc_cache.write() {
if let Some(key) = lowercase(file_path) {
writer.insert(key, calculated_crc);