mirror of
https://github.com/loot/loot-condition-interpreter.git
synced 2026-07-27 14:16:09 -07:00
Speed up CRC32 calculation
It turns out that passing in one byte at a time is really inefficient (which really should have been obvious...). For small files the difference is insignificant, but for Skyrim SE's Update.esm (18 MB) feeding in the buffered file content slice made the calculation twice as fast, and this seems to have non-linear effect, as the benchmark against Dragonborn.esm (63 MB) would have taken ~ 1137 seconds to run when passing one byte at a time (the benchmark using a buffer took ~ 5 seconds). On top of that, switching to the crc32fast crate has an insignificant effect on benchmarking with Blank.esm, but Update.esm's benchmark was twice as fast again, and when benchmarking with Dragonborn.esm, using crc32fast was ~ 170x faster than using the crc crate!
This commit is contained in:
+1
-1
@@ -6,7 +6,7 @@ license = "MIT"
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
crc = "1.0.0"
|
||||
crc32fast = "1.2.0"
|
||||
esplugin = "3.0.0"
|
||||
nom = "5.0.0"
|
||||
pelite = "0.8.0"
|
||||
|
||||
+13
-9
@@ -1,10 +1,9 @@
|
||||
use std::ffi::OsStr;
|
||||
use std::fs::{read_dir, File};
|
||||
use std::hash::Hasher;
|
||||
use std::io::{BufReader, Read};
|
||||
use std::io::{BufRead, BufReader};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crc::{crc32, Hasher32};
|
||||
use regex::Regex;
|
||||
|
||||
use super::{ComparisonOperator, Function};
|
||||
@@ -157,16 +156,21 @@ fn evaluate_checksum(state: &State, file_path: &Path, crc: u32) -> Result<bool,
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let file = File::open(path).map_err(|e| Error::IoError(file_path.to_path_buf(), e))?;
|
||||
let reader = BufReader::new(file);
|
||||
let mut digest = crc32::Digest::new(crc32::IEEE);
|
||||
let io_error_mapper = |e| Error::IoError(file_path.to_path_buf(), e);
|
||||
let file = File::open(path).map_err(io_error_mapper)?;
|
||||
let mut reader = BufReader::new(file);
|
||||
let mut hasher = crc32fast::Hasher::new();
|
||||
|
||||
for byte in reader.bytes() {
|
||||
let byte = byte.map_err(|e| Error::IoError(file_path.to_path_buf(), e))?;
|
||||
digest.write_u8(byte);
|
||||
let mut buffer = reader.fill_buf().map_err(io_error_mapper)?;
|
||||
while !buffer.is_empty() {
|
||||
hasher.write(buffer);
|
||||
let length = buffer.len();
|
||||
reader.consume(length);
|
||||
|
||||
buffer = reader.fill_buf().map_err(io_error_mapper)?;
|
||||
}
|
||||
|
||||
let calculated_crc = digest.sum32();
|
||||
let calculated_crc = hasher.finalize();
|
||||
if let Ok(mut writer) = state.crc_cache.write() {
|
||||
if let Some(key) = lowercase(file_path) {
|
||||
writer.insert(key, calculated_crc);
|
||||
|
||||
Reference in New Issue
Block a user