uutils/coreutils
0
0
Fork 0
mirror of https://github.com/uutils/coreutils.git synced 2026-06-10 15:48:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

base_common: fast_decode improvements

Browse Source 
This is the follow-up commit to the improvement of fast_encode and the
reuse of the read file. What is written there is still valid.

Signed-off-by: Andrea Calabrese <andrea.calabrese@amarulasolutions.com>
This commit is contained in:
Andrea Calabrese
2025-09-10 11:19:05 +02:00
committed by Daniel Hofstetter
parent f39f2a769a
commit f953ccecef
1 changed files with 63 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/uu/base32/src/base_common.rs
+63 -169
View File
@@ -173,9 +173,7 @@ pub fn get_input(config: &Config) -> UResult<Box<dyn ReadSeek>> {
/// Determines if the input buffer ends with padding ('=') after trimming trailing whitespace.
fn read_and_has_padding<R: Read>(input: &mut R) -> UResult<(bool, Vec<u8>)> {
let (start, finish) = input.bytes().size_hint();
let size = finish.unwrap_or(start);
let mut buf = Vec::with_capacity(size);
let mut buf = Vec::new();
input
.read_to_end(&mut buf)
.map_err(|err| USimpleError::new(1, format_read_error(err.kind())))?;
@@ -199,7 +197,7 @@ pub fn handle_input<R: Read + Seek>(input: &mut R, format: Format, config: Confi
let mut stdout_lock = io::stdout().lock();
if config.decode {
fast_decode::fast_decode(
input,
read,
&mut stdout_lock,
supports_fast_decode_and_encode_ref,
config.ignore_garbage,
@@ -293,15 +291,12 @@ pub fn get_supports_fast_decode_and_encode(
pub mod fast_encode {
use crate::base_common::WRAP_DEFAULT;
use std::{
cmp::min,
collections::VecDeque,
io::{self, Write},
num::NonZeroUsize,
cmp::min
};
use uucore::{
encoding::SupportsFastDecodeAndEncode,
error::UResult,
};
use uucore::{encoding::SupportsFastDecodeAndEncode, error::UResult};
struct LineWrapping {
line_length: NonZeroUsize,
@@ -446,38 +441,44 @@ pub mod fast_encode {
let mut encoded_buffer = VecDeque::<u8>::new();
// End of buffers
input.iter().enumerate().step_by(encode_in_chunks_of_size).map(|(idx, _)| {
// The part of `input_buffer` that was actually filled by the call
// to `read`
&input[idx..min(input_size, idx + encode_in_chunks_of_size)]
}).map(|buffer| {
if buffer.len() < encode_in_chunks_of_size - 1 {
leftover_buffer.extend(buffer.into_iter());
assert!(leftover_buffer.len() < encode_in_chunks_of_size);
return None;
}
Some(buffer)
})
.for_each(| buffer| {
if let Some(read_buffer) = buffer {
// Encode data in chunks, then place it in `encoded_buffer`
assert_eq!(read_buffer.len(), encode_in_chunks_of_size);
encode_in_chunks_to_buffer(
supports_fast_decode_and_encode,
read_buffer,
&mut encoded_buffer,
).unwrap();
// Write all data in `encoded_buffer` to `output`
write_to_output(
&mut line_wrapping,
&mut encoded_buffer,
output,
false,
wrap == Some(0),
).unwrap();
}
});
input
.iter()
.enumerate()
.step_by(encode_in_chunks_of_size)
.map(|(idx, _)| {
// The part of `input_buffer` that was actually filled by the call
// to `read`
&input[idx..min(input_size, idx + encode_in_chunks_of_size)]
})
.map(|buffer| {
if buffer.len() < encode_in_chunks_of_size {
leftover_buffer.extend(buffer);
assert!(leftover_buffer.len() < encode_in_chunks_of_size);
return None;
}
Some(buffer)
})
.for_each(|buffer| {
if let Some(read_buffer) = buffer {
// Encode data in chunks, then place it in `encoded_buffer`
assert_eq!(read_buffer.len(), encode_in_chunks_of_size);
encode_in_chunks_to_buffer(
supports_fast_decode_and_encode,
read_buffer,
&mut encoded_buffer,
)
.unwrap();
// Write all data in `encoded_buffer` to `output`
write_to_output(
&mut line_wrapping,
&mut encoded_buffer,
output,
false,
wrap == Some(0),
)
.unwrap();
}
});
// Cleanup
// `input` has finished producing data, so the data remaining in the buffers needs to be encoded and printed
@@ -501,12 +502,8 @@ pub mod fast_encode {
}
pub mod fast_decode {
use crate::base_common::format_read_error;
use std::io::{self, ErrorKind, Read, Write};
use uucore::{
encoding::SupportsFastDecodeAndEncode,
error::{UResult, USimpleError},
};
use std::io::{self, Write};
use uucore::{encoding::SupportsFastDecodeAndEncode, error::UResult};
// Start of helper functions
fn alphabet_to_table(alphabet: &[u8], ignore_garbage: bool) -> [bool; 256] {
@@ -547,46 +544,10 @@ pub mod fast_decode {
fn decode_in_chunks_to_buffer(
supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode,
decode_in_chunks_of_size: usize,
bytes_to_steal: usize,
read_buffer_filtered: &[u8],
decoded_buffer: &mut Vec<u8>,
leftover_buffer: &mut Vec<u8>,
) -> UResult<()> {
let bytes_to_chunk = if bytes_to_steal > 0 {
let (stolen_bytes, rest_of_read_buffer_filtered) =
read_buffer_filtered.split_at(bytes_to_steal);
leftover_buffer.extend(stolen_bytes);
// After appending the stolen bytes to `leftover_buffer`, it should be the right size
assert_eq!(leftover_buffer.len(), decode_in_chunks_of_size);
// Decode the old un-decoded data and the stolen bytes, and add the result to
// `decoded_buffer`
supports_fast_decode_and_encode.decode_into_vec(leftover_buffer, decoded_buffer)?;
// Reset `leftover_buffer`
leftover_buffer.clear();
rest_of_read_buffer_filtered
} else {
// Do not need to steal bytes from `read_buffer`
read_buffer_filtered
};
let chunks_exact = bytes_to_chunk.chunks_exact(decode_in_chunks_of_size);
let remainder = chunks_exact.remainder();
for sl in chunks_exact {
assert_eq!(sl.len(), decode_in_chunks_of_size);
supports_fast_decode_and_encode.decode_into_vec(sl, decoded_buffer)?;
}
leftover_buffer.extend(remainder);
supports_fast_decode_and_encode.decode_into_vec(read_buffer_filtered, decoded_buffer)?;
Ok(())
}
@@ -601,14 +562,11 @@ pub mod fast_decode {
// End of helper functions
pub fn fast_decode(
input: &mut dyn Read,
input: Vec<u8>,
output: &mut dyn Write,
supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode,
ignore_garbage: bool,
) -> UResult<()> {
// Based on performance testing
const INPUT_BUFFER_SIZE: usize = 32 * 1_024;
const DECODE_IN_CHUNKS_OF_SIZE_MULTIPLE: usize = 1_024;
let alphabet = supports_fast_decode_and_encode.alphabet();
@@ -630,102 +588,40 @@ pub mod fast_decode {
let table = alphabet_to_table(alphabet, ignore_garbage);
// Start of buffers
// Data that was read from `input`
let mut input_buffer = vec![0; INPUT_BUFFER_SIZE];
assert!(!input_buffer.is_empty());
// Data that was read from `input` but has not been decoded yet
let mut leftover_buffer = Vec::<u8>::new();
// Decoded data that needs to be written to `output`
let mut decoded_buffer = Vec::<u8>::new();
// Buffer that will be used when `ignore_garbage` is true, and the chunk read from `input` contains garbage
// data
let mut non_garbage_buffer = Vec::<u8>::new();
// End of buffers
loop {
match input.read(&mut input_buffer) {
Ok(bytes_read_from_input) => {
if bytes_read_from_input == 0 {
break;
}
let read_buffer_filtered = {
// The part of `input_buffer` that was actually filled by the call to `read`
let read_buffer = &input_buffer[..bytes_read_from_input];
// First just scan the data for the happy path
// Yields significant speedup when the input does not contain line endings
let found_garbage = read_buffer.iter().any(|ue| {
// Garbage, since it was not found in the table
!table[usize::from(*ue)]
});
if found_garbage {
non_garbage_buffer.clear();
for ue in read_buffer {
if table[usize::from(*ue)] {
// Not garbage, since it was found in the table
non_garbage_buffer.push(*ue);
}
}
non_garbage_buffer.as_slice()
} else {
read_buffer
}
};
// How many bytes to steal from `read_buffer` to get `leftover_buffer` to the right size
let bytes_to_steal = decode_in_chunks_of_size - leftover_buffer.len();
if bytes_to_steal > read_buffer_filtered.len() {
// Do not have enough data to decode a chunk, so copy data to `leftover_buffer` and read more
leftover_buffer.extend(read_buffer_filtered);
assert!(leftover_buffer.len() < decode_in_chunks_of_size);
continue;
}
let mut buffer = Vec::with_capacity(decode_in_chunks_of_size);
input
.iter()
.filter(|ch| table[usize::from(**ch)])
.for_each(|ch| {
buffer.push(*ch);
// How many bytes to steal from `read_buffer` to get
// `leftover_buffer` to the right size
if buffer.len() == decode_in_chunks_of_size {
assert_eq!(decode_in_chunks_of_size, buffer.len());
// Decode data in chunks, then place it in `decoded_buffer`
decode_in_chunks_to_buffer(
supports_fast_decode_and_encode,
decode_in_chunks_of_size,
bytes_to_steal,
read_buffer_filtered,
&buffer,
&mut decoded_buffer,
&mut leftover_buffer,
)?;
assert!(leftover_buffer.len() < decode_in_chunks_of_size);
)
.unwrap();
// Write all data in `decoded_buffer` to `output`
write_to_output(&mut decoded_buffer, output)?;
write_to_output(&mut decoded_buffer, output).unwrap();
buffer.clear();
}
Err(er) => {
let kind = er.kind();
if kind == ErrorKind::Interrupted {
// Retry reading
continue;
}
return Err(USimpleError::new(1, format_read_error(kind)));
}
}
}
});
// Cleanup
// `input` has finished producing data, so the data remaining in the buffers needs to be decoded and printed
{
// Decode all remaining encoded bytes, placing them in `decoded_buffer`
supports_fast_decode_and_encode
.decode_into_vec(&leftover_buffer, &mut decoded_buffer)?;
supports_fast_decode_and_encode.decode_into_vec(&buffer, &mut decoded_buffer)?;
// Write all data in `decoded_buffer` to `output`
write_to_output(&mut decoded_buffer, output)?;
@@ -757,8 +653,6 @@ fn format_read_error(kind: ErrorKind) -> String {
#[cfg(test)]
mod tests {
use crate::base_common::read_and_has_padding;
use super::*;
use std::io::Cursor;
#[test]