use crate::conversion_tables::ConversionTable;
use crate::datastructures::ConversionMode;
use crate::progress::ReadStat;
const NEWLINE: u8 = b'\n';
const SPACE: u8 = b' ';
fn block(buf: &[u8], cbs: usize, sync: bool, rstat: &mut ReadStat) -> Vec<Vec<u8>> {
let mut blocks = buf
.split(|&e| e == NEWLINE)
.map(|split| split.to_vec())
.fold(Vec::new(), |mut blocks, mut split| {
if split.len() > cbs {
rstat.records_truncated += 1;
}
split.resize(cbs, SPACE);
blocks.push(split);
blocks
});
if let Some(last) = blocks.last() {
if (!sync || rstat.reads_partial == 0) && last.iter().all(|&e| e == SPACE) {
blocks.pop();
}
}
blocks
}
fn unblock(buf: &[u8], cbs: usize) -> Vec<u8> {
buf.chunks(cbs).fold(Vec::new(), |mut acc, block| {
if let Some(last_char_idx) = block.iter().rposition(|&e| e != SPACE) {
acc.extend(&block[..=last_char_idx]);
}
acc.push(NEWLINE);
acc
})
}
pub(crate) fn conv_block_unblock_helper(
buf: Vec<u8>,
mode: &ConversionMode,
rstat: &mut ReadStat,
) -> Vec<u8> {
fn apply_conversion(buf: Vec<u8>, ct: &ConversionTable) -> impl Iterator<Item = u8> + '_ {
buf.into_iter().map(|b| ct[b as usize])
}
match mode {
ConversionMode::ConvertOnly(ct) => apply_conversion(buf, ct).collect(),
ConversionMode::BlockThenConvert(ct, cbs, sync) => {
let blocks = block(&buf, *cbs, *sync, rstat);
blocks
.into_iter()
.flat_map(|block| apply_conversion(block, ct))
.collect()
}
ConversionMode::ConvertThenBlock(ct, cbs, sync) => {
let buf: Vec<_> = apply_conversion(buf, ct).collect();
block(&buf, *cbs, *sync, rstat)
.into_iter()
.flatten()
.collect()
}
ConversionMode::BlockOnly(cbs, sync) => block(&buf, *cbs, *sync, rstat)
.into_iter()
.flatten()
.collect(),
ConversionMode::UnblockThenConvert(ct, cbs) => {
let buf = unblock(&buf, *cbs);
apply_conversion(buf, ct).collect()
}
ConversionMode::ConvertThenUnblock(ct, cbs) => {
let buf: Vec<_> = apply_conversion(buf, ct).collect();
unblock(&buf, *cbs)
}
ConversionMode::UnblockOnly(cbs) => unblock(&buf, *cbs),
}
}
#[cfg(test)]
mod tests {
use crate::blocks::{block, unblock};
use crate::progress::ReadStat;
const NEWLINE: u8 = b'\n';
const SPACE: u8 = b' ';
#[test]
fn block_test_no_nl() {
let mut rs = ReadStat::default();
let buf = [0u8, 1u8, 2u8, 3u8];
let res = block(&buf, 4, false, &mut rs);
assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8],]);
}
#[test]
fn block_test_no_nl_short_record() {
let mut rs = ReadStat::default();
let buf = [0u8, 1u8, 2u8, 3u8];
let res = block(&buf, 8, false, &mut rs);
assert_eq!(
res,
vec![vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE],]
);
}
#[test]
fn block_test_no_nl_trunc() {
let mut rs = ReadStat::default();
let buf = [0u8, 1u8, 2u8, 3u8, 4u8];
let res = block(&buf, 4, false, &mut rs);
assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8 ],]);
assert_eq!(rs.records_truncated, 1);
}
#[test]
fn block_test_nl_gt_cbs_trunc() {
let mut rs = ReadStat::default();
let buf = [
0u8, 1u8, 2u8, 3u8, 4u8, NEWLINE, 0u8, 1u8, 2u8, 3u8, 4u8, NEWLINE, 5u8, 6u8, 7u8, 8u8,
];
let res = block(&buf, 4, false, &mut rs);
assert_eq!(
res,
vec![
vec![0u8, 1u8, 2u8, 3u8],
vec![0u8, 1u8, 2u8, 3u8],
vec![5u8, 6u8, 7u8, 8u8],
]
);
assert_eq!(rs.records_truncated, 2);
}
#[test]
fn block_test_surrounded_nl() {
let mut rs = ReadStat::default();
let buf = [0u8, 1u8, 2u8, 3u8, NEWLINE, 4u8, 5u8, 6u8, 7u8, 8u8];
let res = block(&buf, 8, false, &mut rs);
assert_eq!(
res,
vec![
vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE],
vec![4u8, 5u8, 6u8, 7u8, 8u8, SPACE, SPACE, SPACE],
]
);
}
#[test]
fn block_test_multiple_nl_same_cbs_block() {
let mut rs = ReadStat::default();
let buf = [
0u8, 1u8, 2u8, 3u8, NEWLINE, 4u8, NEWLINE, 5u8, 6u8, 7u8, 8u8, 9u8,
];
let res = block(&buf, 8, false, &mut rs);
assert_eq!(
res,
vec![
vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE],
vec![4u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE],
vec![5u8, 6u8, 7u8, 8u8, 9u8, SPACE, SPACE, SPACE],
]
);
}
#[test]
fn block_test_multiple_nl_diff_cbs_block() {
let mut rs = ReadStat::default();
let buf = [
0u8, 1u8, 2u8, 3u8, NEWLINE, 4u8, 5u8, 6u8, 7u8, NEWLINE, 8u8, 9u8,
];
let res = block(&buf, 8, false, &mut rs);
assert_eq!(
res,
vec![
vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE],
vec![4u8, 5u8, 6u8, 7u8, SPACE, SPACE, SPACE, SPACE],
vec![8u8, 9u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE],
]
);
}
#[test]
fn block_test_end_nl_diff_cbs_block() {
let mut rs = ReadStat::default();
let buf = [0u8, 1u8, 2u8, 3u8, NEWLINE];
let res = block(&buf, 4, false, &mut rs);
assert_eq!(res, vec![vec![0u8, 1u8, 2u8, 3u8],]);
}
#[test]
fn block_test_end_nl_same_cbs_block() {
let mut rs = ReadStat::default();
let buf = [0u8, 1u8, 2u8, NEWLINE];
let res = block(&buf, 4, false, &mut rs);
assert_eq!(res, vec![vec![0u8, 1u8, 2u8, SPACE]]);
}
#[test]
fn block_test_double_end_nl() {
let mut rs = ReadStat::default();
let buf = [0u8, 1u8, 2u8, NEWLINE, NEWLINE];
let res = block(&buf, 4, false, &mut rs);
assert_eq!(
res,
vec![vec![0u8, 1u8, 2u8, SPACE], vec![SPACE, SPACE, SPACE, SPACE],]
);
}
#[test]
fn block_test_start_nl() {
let mut rs = ReadStat::default();
let buf = [NEWLINE, 0u8, 1u8, 2u8, 3u8];
let res = block(&buf, 4, false, &mut rs);
assert_eq!(
res,
vec![vec![SPACE, SPACE, SPACE, SPACE], vec![0u8, 1u8, 2u8, 3u8],]
);
}
#[test]
fn block_test_double_surrounded_nl_no_trunc() {
let mut rs = ReadStat::default();
let buf = [0u8, 1u8, 2u8, 3u8, NEWLINE, NEWLINE, 4u8, 5u8, 6u8, 7u8];
let res = block(&buf, 8, false, &mut rs);
assert_eq!(
res,
vec![
vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE],
vec![SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE],
vec![4u8, 5u8, 6u8, 7u8, SPACE, SPACE, SPACE, SPACE],
]
);
}
#[test]
fn block_test_double_surrounded_nl_double_trunc() {
let mut rs = ReadStat::default();
let buf = [
0u8, 1u8, 2u8, 3u8, NEWLINE, NEWLINE, 4u8, 5u8, 6u8, 7u8, 8u8,
];
let res = block(&buf, 4, false, &mut rs);
assert_eq!(
res,
vec![
vec![0u8, 1u8, 2u8, 3u8],
vec![SPACE, SPACE, SPACE, SPACE],
vec![4u8, 5u8, 6u8, 7u8 ],
]
);
assert_eq!(rs.records_truncated, 1);
}
#[test]
fn unblock_test_full_cbs() {
let buf = [0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8];
let res = unblock(&buf, 8);
assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, 4u8, 5u8, 6u8, 7u8, NEWLINE],);
}
#[test]
fn unblock_test_all_space() {
let buf = [SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE];
let res = unblock(&buf, 8);
assert_eq!(res, vec![NEWLINE],);
}
#[test]
fn unblock_test_decoy_spaces() {
let buf = [0u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, 7u8];
let res = unblock(&buf, 8);
assert_eq!(
res,
vec![0u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, 7u8, NEWLINE],
);
}
#[test]
fn unblock_test_strip_single_cbs() {
let buf = [0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE];
let res = unblock(&buf, 8);
assert_eq!(res, vec![0u8, 1u8, 2u8, 3u8, NEWLINE],);
}
#[test]
fn unblock_test_strip_multi_cbs() {
let buf = vec![
vec![0u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE],
vec![0u8, 1u8, SPACE, SPACE, SPACE, SPACE, SPACE, SPACE],
vec![0u8, 1u8, 2u8, SPACE, SPACE, SPACE, SPACE, SPACE],
vec![0u8, 1u8, 2u8, 3u8, SPACE, SPACE, SPACE, SPACE],
]
.into_iter()
.flatten()
.collect::<Vec<_>>();
let res = unblock(&buf, 8);
let exp = vec![
vec![0u8, NEWLINE],
vec![0u8, 1u8, NEWLINE],
vec![0u8, 1u8, 2u8, NEWLINE],
vec![0u8, 1u8, 2u8, 3u8, NEWLINE],
]
.into_iter()
.flatten()
.collect::<Vec<_>>();
assert_eq!(res, exp);
}
}