uutils/coreutils
0
0
Fork 0
mirror of https://github.com/uutils/coreutils.git synced 2026-06-10 15:48:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

base_common: re-using input from has_padding

Browse Source 
Base64: the function has_padding reads the file and then discards it.
The functions fast_encode and fast_decode re-read the file, providing
significant delay in larger files.
This commit  also reduces the amount of computation done inside the
fast_decode function.
In the next commit there will be the fix for the fast_decode function

Signed-off-by: Andrea Calabrese <andrea.calabrese@amarulasolutions.com>
This commit is contained in:
Andrea Calabrese
2025-09-10 11:19:05 +02:00
committed by Daniel Hofstetter
parent 4cf521be49
commit f39f2a769a
1 changed files with 50 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/uu/base32/src/base_common.rs
+50 -111
View File
@@ -8,7 +8,7 @@
use clap::{Arg, ArgAction, Command};
use std::ffi::OsString;
use std::fs::File;
use std::io::{self, ErrorKind, Read, Seek, SeekFrom};
use std::io::{self, ErrorKind, Read, Seek};
use std::path::{Path, PathBuf};
use uucore::display::Quotable;
use uucore::encoding::{
@@ -172,8 +172,10 @@ pub fn get_input(config: &Config) -> UResult<Box<dyn ReadSeek>> {
}
/// Determines if the input buffer ends with padding ('=') after trimming trailing whitespace.
fn has_padding<R: Read + Seek>(input: &mut R) -> UResult<bool> {
let mut buf = Vec::new();
fn read_and_has_padding<R: Read>(input: &mut R) -> UResult<(bool, Vec<u8>)> {
let (start, finish) = input.bytes().size_hint();
let size = finish.unwrap_or(start);
let mut buf = Vec::with_capacity(size);
input
.read_to_end(&mut buf)
.map_err(|err| USimpleError::new(1, format_read_error(err.kind())))?;
@@ -184,20 +186,17 @@ fn has_padding<R: Read + Seek>(input: &mut R) -> UResult<bool> {
.rfind(|&&byte| !byte.is_ascii_whitespace())
.is_some_and(|&byte| byte == b'=');
input.seek(SeekFrom::Start(0))?;
Ok(has_padding)
Ok((has_padding, buf))
}
pub fn handle_input<R: Read + Seek>(input: &mut R, format: Format, config: Config) -> UResult<()> {
let has_padding = has_padding(input)?;
let (has_padding, read) = read_and_has_padding(input)?;
let supports_fast_decode_and_encode =
get_supports_fast_decode_and_encode(format, config.decode, has_padding);
let supports_fast_decode_and_encode_ref = supports_fast_decode_and_encode.as_ref();
let mut stdout_lock = io::stdout().lock();
if config.decode {
fast_decode::fast_decode(
input,
@@ -207,7 +206,7 @@ pub fn handle_input<R: Read + Seek>(input: &mut R, format: Format, config: Confi
)
} else {
fast_encode::fast_encode(
input,
read,
&mut stdout_lock,
supports_fast_decode_and_encode_ref,
config.wrap_cols,
@@ -292,15 +291,16 @@ pub fn get_supports_fast_decode_and_encode(
}
pub mod fast_encode {
use crate::base_common::{WRAP_DEFAULT, format_read_error};
use crate::base_common::WRAP_DEFAULT;
use std::{
collections::VecDeque,
io::{self, ErrorKind, Read, Write},
io::{self, Write},
num::NonZeroUsize,
cmp::min
};
use uucore::{
encoding::SupportsFastDecodeAndEncode,
error::{UResult, USimpleError},
error::UResult,
};
struct LineWrapping {
@@ -311,46 +311,10 @@ pub mod fast_encode {
// Start of helper functions
fn encode_in_chunks_to_buffer(
supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode,
encode_in_chunks_of_size: usize,
bytes_to_steal: usize,
read_buffer: &[u8],
encoded_buffer: &mut VecDeque<u8>,
leftover_buffer: &mut VecDeque<u8>,
) -> UResult<()> {
let bytes_to_chunk = if bytes_to_steal > 0 {
let (stolen_bytes, rest_of_read_buffer) = read_buffer.split_at(bytes_to_steal);
leftover_buffer.extend(stolen_bytes);
// After appending the stolen bytes to `leftover_buffer`, it should be the right size
assert_eq!(leftover_buffer.len(), encode_in_chunks_of_size);
// Encode the old unencoded data and the stolen bytes, and add the result to
// `encoded_buffer`
supports_fast_decode_and_encode
.encode_to_vec_deque(leftover_buffer.make_contiguous(), encoded_buffer)?;
// Reset `leftover_buffer`
leftover_buffer.clear();
rest_of_read_buffer
} else {
// Do not need to steal bytes from `read_buffer`
read_buffer
};
let chunks_exact = bytes_to_chunk.chunks_exact(encode_in_chunks_of_size);
let remainder = chunks_exact.remainder();
for sl in chunks_exact {
assert_eq!(sl.len(), encode_in_chunks_of_size);
supports_fast_decode_and_encode.encode_to_vec_deque(sl, encoded_buffer)?;
}
leftover_buffer.extend(remainder);
supports_fast_decode_and_encode.encode_to_vec_deque(read_buffer, encoded_buffer)?;
Ok(())
}
@@ -440,13 +404,12 @@ pub mod fast_encode {
// End of helper functions
pub fn fast_encode(
input: &mut dyn Read,
input: Vec<u8>,
output: &mut dyn Write,
supports_fast_decode_and_encode: &dyn SupportsFastDecodeAndEncode,
wrap: Option<usize>,
) -> UResult<()> {
// Based on performance testing
const INPUT_BUFFER_SIZE: usize = 32 * 1_024;
const ENCODE_IN_CHUNKS_OF_SIZE_MULTIPLE: usize = 1_024;
@@ -473,12 +436,9 @@ pub mod fast_encode {
}),
};
let input_size = input.len();
// Start of buffers
// Data that was read from `input`
let mut input_buffer = vec![0; INPUT_BUFFER_SIZE];
assert!(!input_buffer.is_empty());
// Data that was read from `input` but has not been encoded yet
let mut leftover_buffer = VecDeque::<u8>::new();
@@ -486,60 +446,38 @@ pub mod fast_encode {
let mut encoded_buffer = VecDeque::<u8>::new();
// End of buffers
loop {
match input.read(&mut input_buffer) {
Ok(bytes_read_from_input) => {
if bytes_read_from_input == 0 {
break;
}
// The part of `input_buffer` that was actually filled by the call to `read`
let read_buffer = &input_buffer[..bytes_read_from_input];
// How many bytes to steal from `read_buffer` to get `leftover_buffer` to the right size
let bytes_to_steal = encode_in_chunks_of_size - leftover_buffer.len();
if bytes_to_steal > bytes_read_from_input {
// Do not have enough data to encode a chunk, so copy data to `leftover_buffer` and read more
leftover_buffer.extend(read_buffer);
assert!(leftover_buffer.len() < encode_in_chunks_of_size);
continue;
}
// Encode data in chunks, then place it in `encoded_buffer`
encode_in_chunks_to_buffer(
supports_fast_decode_and_encode,
encode_in_chunks_of_size,
bytes_to_steal,
read_buffer,
&mut encoded_buffer,
&mut leftover_buffer,
)?;
assert!(leftover_buffer.len() < encode_in_chunks_of_size);
// Write all data in `encoded_buffer` to `output`
write_to_output(
&mut line_wrapping,
&mut encoded_buffer,
output,
false,
wrap == Some(0),
)?;
}
Err(er) => {
let kind = er.kind();
if kind == ErrorKind::Interrupted {
// Retry reading
continue;
}
return Err(USimpleError::new(1, format_read_error(kind)));
}
input.iter().enumerate().step_by(encode_in_chunks_of_size).map(|(idx, _)| {
// The part of `input_buffer` that was actually filled by the call
// to `read`
&input[idx..min(input_size, idx + encode_in_chunks_of_size)]
}).map(|buffer| {
if buffer.len() < encode_in_chunks_of_size - 1 {
leftover_buffer.extend(buffer.into_iter());
assert!(leftover_buffer.len() < encode_in_chunks_of_size);
return None;
}
}
Some(buffer)
})
.for_each(| buffer| {
if let Some(read_buffer) = buffer {
// Encode data in chunks, then place it in `encoded_buffer`
assert_eq!(read_buffer.len(), encode_in_chunks_of_size);
encode_in_chunks_to_buffer(
supports_fast_decode_and_encode,
read_buffer,
&mut encoded_buffer,
).unwrap();
// Write all data in `encoded_buffer` to `output`
write_to_output(
&mut line_wrapping,
&mut encoded_buffer,
output,
false,
wrap == Some(0),
).unwrap();
}
});
// Cleanup
// `input` has finished producing data, so the data remaining in the buffers needs to be encoded and printed
@@ -558,7 +496,6 @@ pub mod fast_encode {
wrap == Some(0),
)?;
}
Ok(())
}
}
@@ -819,6 +756,8 @@ fn format_read_error(kind: ErrorKind) -> String {
#[cfg(test)]
mod tests {
use crate::base_common::read_and_has_padding;
use super::*;
use std::io::Cursor;
@@ -838,7 +777,7 @@ mod tests {
for (input, expected) in test_cases {
let mut cursor = Cursor::new(input.as_bytes());
assert_eq!(
has_padding(&mut cursor).unwrap(),
read_and_has_padding(&mut cursor).unwrap().0,
expected,
"Failed for input: '{input}'"
);