uutils/coreutils
0
0
Fork 0
mirror of https://github.com/uutils/coreutils.git synced 2026-06-10 15:48:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8386 from RenjiSann/improve-sort

Browse Source 
sort: Handle non-utf8 sorting content
This commit is contained in:
Daniel Hofstetter
2025-07-25 14:52:05 +02:00
committed by GitHub
parent 98374e8021 27faee711b
commit 8bbce0d3de
15 changed files with 369 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/uu/ls/src/ls.rs
+5 -2
View File
@@ -2139,8 +2139,11 @@ fn sort_entries(entries: &mut [PathData], config: &Config, out: &mut BufWriter<S
// The default sort in GNU ls is case insensitive
Sort::Name => entries.sort_by(|a, b| a.display_name.cmp(&b.display_name)),
Sort::Version => entries.sort_by(|a, b| {
version_cmp(&a.p_buf.to_string_lossy(), &b.p_buf.to_string_lossy())
.then(a.p_buf.to_string_lossy().cmp(&b.p_buf.to_string_lossy()))
version_cmp(
os_str_as_bytes_lossy(a.p_buf.as_os_str()).as_ref(),
os_str_as_bytes_lossy(b.p_buf.as_os_str()).as_ref(),
)
.then(a.p_buf.to_string_lossy().cmp(&b.p_buf.to_string_lossy()))
}),
Sort::Extension => entries.sort_by(|a, b| {
a.p_buf
src/uu/sort/src/check.rs
+2 -2
View File
@@ -72,7 +72,7 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> {
return Err(SortError::Disorder {
file: path.to_owned(),
line_number: line_idx,
line: new_first.line.to_owned(),
line: String::from_utf8_lossy(new_first.line).into_owned(),
silent: settings.check_silent,
}
.into());
@@ -86,7 +86,7 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> {
return Err(SortError::Disorder {
file: path.to_owned(),
line_number: line_idx,
line: b.line.to_owned(),
line: String::from_utf8_lossy(b.line).into_owned(),
silent: settings.check_silent,
}
.into());
src/uu/sort/src/chunks.rs
+9 -12
View File
@@ -17,9 +17,7 @@ use memchr::memchr_iter;
use self_cell::self_cell;
use uucore::error::{UResult, USimpleError};
use crate::{
GeneralBigDecimalParseResult, GlobalSettings, Line, SortError, numeric_str_cmp::NumInfo,
};
use crate::{GeneralBigDecimalParseResult, GlobalSettings, Line, numeric_str_cmp::NumInfo};
self_cell!(
/// The chunk that is passed around between threads.
@@ -41,7 +39,7 @@ pub struct ChunkContents<'a> {
#[derive(Debug)]
pub struct LineData<'a> {
pub selections: Vec<&'a str>,
pub selections: Vec<&'a [u8]>,
pub num_infos: Vec<NumInfo>,
pub parsed_floats: Vec<GeneralBigDecimalParseResult>,
pub line_num_floats: Vec<Option<f64>>,
@@ -68,7 +66,7 @@ impl Chunk {
let selections = unsafe {
// SAFETY: (same as above) It is safe to (temporarily) transmute to a vector of &str with a longer lifetime,
// because the vector is empty.
std::mem::transmute::<Vec<&'_ str>, Vec<&'static str>>(std::mem::take(
std::mem::transmute::<Vec<&'_ [u8]>, Vec<&'static [u8]>>(std::mem::take(
&mut contents.line_data.selections,
))
};
@@ -100,7 +98,7 @@ impl Chunk {
pub struct RecycledChunk {
lines: Vec<Line<'static>>,
selections: Vec<&'static str>,
selections: Vec<&'static [u8]>,
num_infos: Vec<NumInfo>,
parsed_floats: Vec<GeneralBigDecimalParseResult>,
line_num_floats: Vec<Option<f64>>,
@@ -180,15 +178,14 @@ pub fn read<T: Read>(
let selections = unsafe {
// SAFETY: It is safe to transmute to an empty vector of selections with shorter lifetime.
// It was only temporarily transmuted to a Vec<Line<'static>> to make recycling possible.
std::mem::transmute::<Vec<&'static str>, Vec<&'_ str>>(selections)
std::mem::transmute::<Vec<&'static [u8]>, Vec<&'_ [u8]>>(selections)
};
let mut lines = unsafe {
// SAFETY: (same as above) It is safe to transmute to a vector of lines with shorter lifetime,
// because it was only temporarily transmuted to a Vec<Line<'static>> to make recycling possible.
std::mem::transmute::<Vec<Line<'static>>, Vec<Line<'_>>>(lines)
};
let read = std::str::from_utf8(&buffer[..read])
.map_err(|error| SortError::Uft8Error { error })?;
let read = &buffer[..read];
let mut line_data = LineData {
selections,
num_infos,
@@ -205,13 +202,13 @@ pub fn read<T: Read>(
/// Split `read` into `Line`s, and add them to `lines`.
fn parse_lines<'a>(
read: &'a str,
read: &'a [u8],
lines: &mut Vec<Line<'a>>,
line_data: &mut LineData<'a>,
separator: u8,
settings: &GlobalSettings,
) {
let read = read.strip_suffix(separator as char).unwrap_or(read);
let read = read.strip_suffix(&[separator]).unwrap_or(read);
assert!(lines.is_empty());
assert!(line_data.selections.is_empty());
@@ -220,7 +217,7 @@ fn parse_lines<'a>(
assert!(line_data.line_num_floats.is_empty());
let mut token_buffer = vec![];
lines.extend(
read.split(separator as char)
read.split(|&c| c == separator)
.enumerate()
.map(|(index, line)| Line::create(line, index, line_data, &mut token_buffer, settings)),
);
src/uu/sort/src/custom_str_cmp.rs
+9 -9
View File
@@ -9,7 +9,7 @@
use std::cmp::Ordering;
fn filter_char(c: char, ignore_non_printing: bool, ignore_non_dictionary: bool) -> bool {
fn filter_char(c: u8, ignore_non_printing: bool, ignore_non_dictionary: bool) -> bool {
if ignore_non_dictionary && !(c.is_ascii_alphanumeric() || c.is_ascii_whitespace()) {
return false;
}
@@ -19,7 +19,7 @@ fn filter_char(c: char, ignore_non_printing: bool, ignore_non_dictionary: bool)
true
}
fn cmp_chars(a: char, b: char, ignore_case: bool) -> Ordering {
fn cmp_chars(a: u8, b: u8, ignore_case: bool) -> Ordering {
if ignore_case {
a.to_ascii_uppercase().cmp(&b.to_ascii_uppercase())
} else {
@@ -28,8 +28,8 @@ fn cmp_chars(a: char, b: char, ignore_case: bool) -> Ordering {
}
pub fn custom_str_cmp(
a: &str,
b: &str,
a: &[u8],
b: &[u8],
ignore_non_printing: bool,
ignore_non_dictionary: bool,
ignore_case: bool,
@@ -39,11 +39,11 @@ pub fn custom_str_cmp(
return a.cmp(b);
}
let mut a_chars = a
.chars()
.filter(|&c| filter_char(c, ignore_non_printing, ignore_non_dictionary));
.iter()
.filter(|&&c| filter_char(c, ignore_non_printing, ignore_non_dictionary));
let mut b_chars = b
.chars()
.filter(|&c| filter_char(c, ignore_non_printing, ignore_non_dictionary));
.iter()
.filter(|&&c| filter_char(c, ignore_non_printing, ignore_non_dictionary));
loop {
let a_char = a_chars.next();
let b_char = b_chars.next();
@@ -52,7 +52,7 @@ pub fn custom_str_cmp(
(Some(_), None) => return Ordering::Greater,
(None, Some(_)) => return Ordering::Less,
(Some(a_char), Some(b_char)) => {
let ordering = cmp_chars(a_char, b_char, ignore_case);
let ordering = cmp_chars(*a_char, *b_char, ignore_case);
if ordering != Ordering::Equal {
return ordering;
}
src/uu/sort/src/ext_sort.rs
+1 -1
View File
@@ -272,7 +272,7 @@ fn write<I: WriteableTmpFile>(
fn write_lines<T: Write>(lines: &[Line], writer: &mut T, separator: u8) {
for s in lines {
writer.write_all(s.line.as_bytes()).unwrap();
writer.write_all(s.line).unwrap();
writer.write_all(&[separator]).unwrap();
}
}
src/uu/sort/src/numeric_str_cmp.rs
+83 -74
View File
@@ -28,8 +28,8 @@ pub struct NumInfo {
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct NumInfoParseSettings {
pub accept_si_units: bool,
pub thousands_separator: Option<char>,
pub decimal_pt: Option<char>,
pub thousands_separator: Option<u8>,
pub decimal_pt: Option<u8>,
}
impl Default for NumInfoParseSettings {
@@ -37,7 +37,7 @@ impl Default for NumInfoParseSettings {
Self {
accept_si_units: false,
thousands_separator: None,
decimal_pt: Some('.'),
decimal_pt: Some(b'.'),
}
}
}
@@ -51,7 +51,7 @@ impl NumInfo {
/// If the input is not a number (which has to be treated as zero), the returned empty range
/// will be 0..0.
#[allow(clippy::cognitive_complexity)]
pub fn parse(num: &str, parse_settings: &NumInfoParseSettings) -> (Self, Range<usize>) {
pub fn parse(num: &[u8], parse_settings: &NumInfoParseSettings) -> (Self, Range<usize>) {
let mut exponent = -1;
let mut had_decimal_pt = false;
let mut had_digit = false;
@@ -60,12 +60,12 @@ impl NumInfo {
let mut first_char = true;
for (idx, char) in num.char_indices() {
if first_char && char.is_whitespace() {
for (idx, &char) in num.iter().enumerate() {
if first_char && char.is_ascii_whitespace() {
continue;
}
if first_char && char == '-' {
if first_char && char == b'-' {
sign = Sign::Negative;
first_char = false;
continue;
@@ -84,7 +84,16 @@ impl NumInfo {
let has_si_unit = parse_settings.accept_si_units
&& matches!(
char,
'K' | 'k' | 'M' | 'G' | 'T' | 'P' | 'E' | 'Z' | 'Y' | 'R' | 'Q'
b'K' | b'k'
| b'M'
| b'G'
| b'T'
| b'P'
| b'E'
| b'Z'
| b'Y'
| b'R'
| b'Q'
);
(
Self { exponent, sign },
@@ -112,7 +121,7 @@ impl NumInfo {
continue;
}
had_digit = true;
if start.is_none() && char == '0' {
if start.is_none() && char == b'0' {
if had_decimal_pt {
// We're parsing a number whose first nonzero digit is after the decimal point.
exponent -= 1;
@@ -124,7 +133,7 @@ impl NumInfo {
if !had_decimal_pt {
exponent += 1;
}
if start.is_none() && char != '0' {
if start.is_none() && char != b'0' {
start = Some(idx);
}
}
@@ -150,7 +159,7 @@ impl NumInfo {
}
fn is_invalid_char(
c: char,
c: u8,
had_decimal_pt: &mut bool,
parse_settings: &NumInfoParseSettings,
) -> bool {
@@ -168,19 +177,19 @@ impl NumInfo {
}
}
fn get_unit(unit: Option<char>) -> u8 {
fn get_unit(unit: Option<u8>) -> u8 {
if let Some(unit) = unit {
match unit {
'K' | 'k' => 1,
'M' => 2,
'G' => 3,
'T' => 4,
'P' => 5,
'E' => 6,
'Z' => 7,
'Y' => 8,
'R' => 9,
'Q' => 10,
b'K' | b'k' => 1,
b'M' => 2,
b'G' => 3,
b'T' => 4,
b'P' => 5,
b'E' => 6,
b'Z' => 7,
b'Y' => 8,
b'R' => 9,
b'Q' => 10,
_ => 0,
}
} else {
@@ -191,16 +200,16 @@ fn get_unit(unit: Option<char>) -> u8 {
/// Compare two numbers according to the rules of human numeric comparison.
/// The SI-Unit takes precedence over the actual value (i.e. 2000M < 1G).
pub fn human_numeric_str_cmp(
(a, a_info): (&str, &NumInfo),
(b, b_info): (&str, &NumInfo),
(a, a_info): (&[u8], &NumInfo),
(b, b_info): (&[u8], &NumInfo),
) -> Ordering {
// 1. Sign
if a_info.sign != b_info.sign {
return a_info.sign.cmp(&b_info.sign);
}
// 2. Unit
let a_unit = get_unit(a.chars().next_back());
let b_unit = get_unit(b.chars().next_back());
let a_unit = get_unit(a.iter().next_back().copied());
let b_unit = get_unit(b.iter().next_back().copied());
let ordering = a_unit.cmp(&b_unit);
if ordering == Ordering::Equal {
// 3. Number
@@ -215,7 +224,7 @@ pub fn human_numeric_str_cmp(
/// Compare two numbers as strings without parsing them as a number first. This should be more performant and can handle numbers more precisely.
/// [`NumInfo`] is needed to provide a fast path for most numbers.
#[inline(always)]
pub fn numeric_str_cmp((a, a_info): (&str, &NumInfo), (b, b_info): (&str, &NumInfo)) -> Ordering {
pub fn numeric_str_cmp((a, a_info): (&[u8], &NumInfo), (b, b_info): (&[u8], &NumInfo)) -> Ordering {
// check for a difference in the sign
if a_info.sign != b_info.sign {
return a_info.sign.cmp(&b_info.sign);
@@ -226,22 +235,22 @@ pub fn numeric_str_cmp((a, a_info): (&str, &NumInfo), (b, b_info): (&str, &NumIn
a_info.exponent.cmp(&b_info.exponent)
} else {
// walk the characters from the front until we find a difference
let mut a_chars = a.chars().filter(char::is_ascii_digit);
let mut b_chars = b.chars().filter(char::is_ascii_digit);
let mut a_chars = a.iter().copied().filter(u8::is_ascii_digit);
let mut b_chars = b.iter().copied().filter(u8::is_ascii_digit);
loop {
let a_next = a_chars.next();
let b_next = b_chars.next();
match (a_next, b_next) {
(None, None) => break Ordering::Equal,
(Some(c), None) => {
break if c == '0' && a_chars.all(|c| c == '0') {
break if c == b'0' && a_chars.all(|c| c == b'0') {
Ordering::Equal
} else {
Ordering::Greater
};
}
(None, Some(c)) => {
break if c == '0' && b_chars.all(|c| c == '0') {
break if c == b'0' && b_chars.all(|c| c == b'0') {
Ordering::Equal
} else {
Ordering::Less
@@ -270,7 +279,7 @@ mod tests {
#[test]
fn parses_exp() {
let n = "1";
let n = b"1";
assert_eq!(
NumInfo::parse(n, &NumInfoParseSettings::default()),
(
@@ -281,7 +290,7 @@ mod tests {
0..1
)
);
let n = "100";
let n = b"100";
assert_eq!(
NumInfo::parse(n, &NumInfoParseSettings::default()),
(
@@ -292,12 +301,12 @@ mod tests {
0..3
)
);
let n = "1,000";
let n = b"1,000";
assert_eq!(
NumInfo::parse(
n,
&NumInfoParseSettings {
thousands_separator: Some(','),
thousands_separator: Some(b','),
..Default::default()
}
),
@@ -309,7 +318,7 @@ mod tests {
0..5
)
);
let n = "1,000";
let n = b"1,000";
assert_eq!(
NumInfo::parse(n, &NumInfoParseSettings::default()),
(
@@ -320,7 +329,7 @@ mod tests {
0..1
)
);
let n = "1000.00";
let n = b"1000.00";
assert_eq!(
NumInfo::parse(n, &NumInfoParseSettings::default()),
(
@@ -334,7 +343,7 @@ mod tests {
}
#[test]
fn parses_negative_exp() {
let n = "0.00005";
let n = b"0.00005";
assert_eq!(
NumInfo::parse(n, &NumInfoParseSettings::default()),
(
@@ -345,7 +354,7 @@ mod tests {
6..7
)
);
let n = "00000.00005";
let n = b"00000.00005";
assert_eq!(
NumInfo::parse(n, &NumInfoParseSettings::default()),
(
@@ -360,7 +369,7 @@ mod tests {
#[test]
fn parses_sign() {
let n = "5";
let n = b"5";
assert_eq!(
NumInfo::parse(n, &NumInfoParseSettings::default()),
(
@@ -371,7 +380,7 @@ mod tests {
0..1
)
);
let n = "-5";
let n = b"-5";
assert_eq!(
NumInfo::parse(n, &NumInfoParseSettings::default()),
(
@@ -382,7 +391,7 @@ mod tests {
1..2
)
);
let n = " -5";
let n = b" -5";
assert_eq!(
NumInfo::parse(n, &NumInfoParseSettings::default()),
(
@@ -395,7 +404,7 @@ mod tests {
);
}
fn test_helper(a: &str, b: &str, expected: Ordering) {
fn test_helper(a: &[u8], b: &[u8], expected: Ordering) {
let (a_info, a_range) = NumInfo::parse(a, &NumInfoParseSettings::default());
let (b_info, b_range) = NumInfo::parse(b, &NumInfoParseSettings::default());
let ordering = numeric_str_cmp(
@@ -408,71 +417,71 @@ mod tests {
}
#[test]
fn test_single_digit() {
test_helper("1", "2", Ordering::Less);
test_helper("0", "0", Ordering::Equal);
test_helper(b"1", b"2", Ordering::Less);
test_helper(b"0", b"0", Ordering::Equal);
}
#[test]
fn test_minus() {
test_helper("-1", "-2", Ordering::Greater);
test_helper("-0", "-0", Ordering::Equal);
test_helper(b"-1", b"-2", Ordering::Greater);
test_helper(b"-0", b"-0", Ordering::Equal);
}
#[test]
fn test_different_len() {
test_helper("-20", "-100", Ordering::Greater);
test_helper("10.0", "2.000000", Ordering::Greater);
test_helper(b"-20", b"-100", Ordering::Greater);
test_helper(b"10.0", b"2.000000", Ordering::Greater);
}
#[test]
fn test_decimal_digits() {
test_helper("20.1", "20.2", Ordering::Less);
test_helper("20.1", "20.15", Ordering::Less);
test_helper("-20.1", "+20.15", Ordering::Less);
test_helper("-20.1", "-20", Ordering::Less);
test_helper(b"20.1", b"20.2", Ordering::Less);
test_helper(b"20.1", b"20.15", Ordering::Less);
test_helper(b"-20.1", b"+20.15", Ordering::Less);
test_helper(b"-20.1", b"-20", Ordering::Less);
}
#[test]
fn test_trailing_zeroes() {
test_helper("20.00000", "20.1", Ordering::Less);
test_helper("20.00000", "20.0", Ordering::Equal);
test_helper(b"20.00000", b"20.1", Ordering::Less);
test_helper(b"20.00000", b"20.0", Ordering::Equal);
}
#[test]
fn test_invalid_digits() {
test_helper("foo", "bar", Ordering::Equal);
test_helper("20.1", "a", Ordering::Greater);
test_helper("-20.1", "a", Ordering::Less);
test_helper("a", "0.15", Ordering::Less);
test_helper(b"foo", b"bar", Ordering::Equal);
test_helper(b"20.1", b"a", Ordering::Greater);
test_helper(b"-20.1", b"a", Ordering::Less);
test_helper(b"a", b"0.15", Ordering::Less);
}
#[test]
fn test_multiple_decimal_pts() {
test_helper("10.0.0", "50.0.0", Ordering::Less);
test_helper("0.1.", "0.2.0", Ordering::Less);
test_helper("1.1.", "0", Ordering::Greater);
test_helper("1.1.", "-0", Ordering::Greater);
test_helper(b"10.0.0", b"50.0.0", Ordering::Less);
test_helper(b"0.1.", b"0.2.0", Ordering::Less);
test_helper(b"1.1.", b"0", Ordering::Greater);
test_helper(b"1.1.", b"-0", Ordering::Greater);
}
#[test]
fn test_leading_decimal_pts() {
test_helper(".0", ".0", Ordering::Equal);
test_helper(".1", ".0", Ordering::Greater);
test_helper(".02", "0", Ordering::Greater);
test_helper(b".0", b".0", Ordering::Equal);
test_helper(b".1", b".0", Ordering::Greater);
test_helper(b".02", b"0", Ordering::Greater);
}
#[test]
fn test_leading_zeroes() {
test_helper("000000.0", ".0", Ordering::Equal);
test_helper("0.1", "0000000000000.0", Ordering::Greater);
test_helper("-01", "-2", Ordering::Greater);
test_helper(b"000000.0", b".0", Ordering::Equal);
test_helper(b"0.1", b"0000000000000.0", Ordering::Greater);
test_helper(b"-01", b"-2", Ordering::Greater);
}
#[test]
fn minus_zero() {
// This matches GNU sort behavior.
test_helper("-0", "0", Ordering::Equal);
test_helper("-0x", "0", Ordering::Equal);
test_helper(b"-0", b"0", Ordering::Equal);
test_helper(b"-0x", b"0", Ordering::Equal);
}
#[test]
fn double_minus() {
test_helper("--1", "0", Ordering::Equal);
test_helper(b"--1", b"0", Ordering::Equal);
}
#[test]
fn single_minus() {
let info = NumInfo::parse("-", &NumInfoParseSettings::default());
let info = NumInfo::parse(b"-", &NumInfoParseSettings::default());
assert_eq!(
info,
(
@@ -487,7 +496,7 @@ mod tests {
#[test]
fn invalid_with_unit() {
let info = NumInfo::parse(
"-K",
b"-K",
&NumInfoParseSettings {
accept_si_units: true,
..Default::default()
src/uu/sort/src/sort.rs
+150 -172
View File
File diff suppressed because it is too large Load Diff
src/uu/sort/src/tmp_dir.rs
+8 -6
View File
@@ -60,21 +60,23 @@ impl TmpDirWrapper {
// and the program doesn't terminate before the handler has finished
let _lock = lock.lock().unwrap();
if let Err(e) = remove_tmp_dir(&path) {
let mut args = HashMap::new();
args.insert("error".to_string(), e.to_string());
show_error!(
"{}",
get_message_with_args("sort-failed-to-delete-temporary-directory", args)
get_message_with_args(
"sort-failed-to-delete-temporary-directory",
HashMap::from([("error".to_string(), e.to_string())])
)
);
}
std::process::exit(2)
})
.map_err(|e| {
let mut args = HashMap::new();
args.insert("error".to_string(), e.to_string());
USimpleError::new(
2,
get_message_with_args("sort-failed-to-set-up-signal-handler", args),
get_message_with_args(
"sort-failed-to-set-up-signal-handler",
HashMap::from([("error".to_string(), e.to_string())]),
),
)
})
}
src/uucore/src/lib/features/version_cmp.rs
+72 -60
View File
@@ -10,15 +10,15 @@ use std::cmp::Ordering;
/// Compares the non-digit parts of a version.
/// Special cases: ~ are before everything else, even ends ("a~" < "a")
/// Letters are before non-letters
fn version_non_digit_cmp(a: &str, b: &str) -> Ordering {
let mut a_chars = a.chars();
let mut b_chars = b.chars();
fn version_non_digit_cmp(a: &[u8], b: &[u8]) -> Ordering {
let mut a_chars = a.iter();
let mut b_chars = b.iter();
loop {
match (a_chars.next(), b_chars.next()) {
(Some(c1), Some(c2)) if c1 == c2 => {}
(None, None) => return Ordering::Equal,
(_, Some('~')) => return Ordering::Greater,
(Some('~'), _) => return Ordering::Less,
(_, Some(b'~')) => return Ordering::Greater,
(Some(b'~'), _) => return Ordering::Less,
(None, Some(_)) => return Ordering::Less,
(Some(_), None) => return Ordering::Greater,
(Some(c1), Some(c2)) if c1.is_ascii_alphabetic() && !c2.is_ascii_alphabetic() => {
@@ -27,27 +27,27 @@ fn version_non_digit_cmp(a: &str, b: &str) -> Ordering {
(Some(c1), Some(c2)) if !c1.is_ascii_alphabetic() && c2.is_ascii_alphabetic() => {
return Ordering::Greater;
}
(Some(c1), Some(c2)) => return c1.cmp(&c2),
(Some(c1), Some(c2)) => return c1.cmp(c2),
}
}
}
/// Remove file endings matching the regex (\.[A-Za-z~][A-Za-z0-9~]*)*$
fn remove_file_ending(a: &str) -> &str {
fn remove_file_ending(a: &[u8]) -> &[u8] {
let mut ending_start = None;
let mut prev_was_dot = false;
for (idx, char) in a.char_indices() {
if char == '.' {
for (idx, &char) in a.iter().enumerate() {
if char == b'.' {
if ending_start.is_none() || prev_was_dot {
ending_start = Some(idx);
}
prev_was_dot = true;
} else if prev_was_dot {
prev_was_dot = false;
if !char.is_ascii_alphabetic() && char != '~' {
if !char.is_ascii_alphabetic() && char != b'~' {
ending_start = None;
}
} else if !char.is_ascii_alphanumeric() && char != '~' {
} else if !char.is_ascii_alphanumeric() && char != b'~' {
ending_start = None;
}
}
@@ -62,7 +62,7 @@ fn remove_file_ending(a: &str) -> &str {
}
/// Compare two version strings.
pub fn version_cmp(mut a: &str, mut b: &str) -> Ordering {
pub fn version_cmp(mut a: &[u8], mut b: &[u8]) -> Ordering {
let str_cmp = a.cmp(b);
if str_cmp == Ordering::Equal {
return str_cmp;
@@ -77,21 +77,21 @@ pub fn version_cmp(mut a: &str, mut b: &str) -> Ordering {
(false, false) => {}
}
// 2. Dots
match (a == ".", b == ".") {
match (a == b".", b == b".") {
(true, false) => return Ordering::Less,
(false, true) => return Ordering::Greater,
(true, true) => unreachable!(),
(false, false) => {}
}
// 3. Two Dots
match (a == "..", b == "..") {
match (a == b"..", b == b"..") {
(true, false) => return Ordering::Less,
(false, true) => return Ordering::Greater,
(true, true) => unreachable!(),
(false, false) => {}
}
// 4. Strings starting with a dot
match (a.starts_with('.'), b.starts_with('.')) {
match (a.starts_with(b"."), b.starts_with(b".")) {
(true, false) => return Ordering::Less,
(false, true) => return Ordering::Greater,
(true, true) => {
@@ -115,8 +115,8 @@ pub fn version_cmp(mut a: &str, mut b: &str) -> Ordering {
// 2. Compare leading numerical part
// 3. Repeat
while !a.is_empty() || !b.is_empty() {
let a_numerical_start = a.find(|c: char| c.is_ascii_digit()).unwrap_or(a.len());
let b_numerical_start = b.find(|c: char| c.is_ascii_digit()).unwrap_or(b.len());
let a_numerical_start = a.iter().position(|c| c.is_ascii_digit()).unwrap_or(a.len());
let b_numerical_start = b.iter().position(|c| c.is_ascii_digit()).unwrap_or(b.len());
let a_str = &a[..a_numerical_start];
let b_str = &b[..b_numerical_start];
@@ -129,11 +129,17 @@ pub fn version_cmp(mut a: &str, mut b: &str) -> Ordering {
a = &a[a_numerical_start..];
b = &b[a_numerical_start..];
let a_numerical_end = a.find(|c: char| !c.is_ascii_digit()).unwrap_or(a.len());
let b_numerical_end = b.find(|c: char| !c.is_ascii_digit()).unwrap_or(b.len());
let a_numerical_end = a
.iter()
.position(|c| !c.is_ascii_digit())
.unwrap_or(a.len());
let b_numerical_end = b
.iter()
.position(|c| !c.is_ascii_digit())
.unwrap_or(b.len());
let a_str = a[..a_numerical_end].trim_start_matches('0');
let b_str = b[..b_numerical_end].trim_start_matches('0');
let a_str = &a[a.iter().position(|&c| c != b'0').unwrap_or(a.len())..a_numerical_end];
let b_str = &b[b.iter().position(|&c| c != b'0').unwrap_or(b.len())..b_numerical_end];
match a_str.len().cmp(&b_str.len()) {
Ordering::Equal => {}
@@ -159,138 +165,138 @@ mod tests {
#[test]
fn test_version_cmp() {
// Identical strings
assert_eq!(version_cmp("hello", "hello"), Ordering::Equal);
assert_eq!(version_cmp(b"hello", b"hello"), Ordering::Equal);
assert_eq!(version_cmp("file12", "file12"), Ordering::Equal);
assert_eq!(version_cmp(b"file12", b"file12"), Ordering::Equal);
assert_eq!(
version_cmp("file12-suffix", "file12-suffix"),
version_cmp(b"file12-suffix", b"file12-suffix"),
Ordering::Equal
);
assert_eq!(
version_cmp("file12-suffix24", "file12-suffix24"),
version_cmp(b"file12-suffix24", b"file12-suffix24"),
Ordering::Equal
);
// Shortened names
assert_eq!(version_cmp("world", "wo"), Ordering::Greater);
assert_eq!(version_cmp(b"world", b"wo"), Ordering::Greater);
assert_eq!(version_cmp("hello10wo", "hello10world"), Ordering::Less);
assert_eq!(version_cmp(b"hello10wo", b"hello10world"), Ordering::Less);
// Simple names
assert_eq!(version_cmp("world", "hello"), Ordering::Greater);
assert_eq!(version_cmp(b"world", b"hello"), Ordering::Greater);
assert_eq!(version_cmp("hello", "world"), Ordering::Less);
assert_eq!(version_cmp(b"hello", b"world"), Ordering::Less);
assert_eq!(version_cmp("apple", "ant"), Ordering::Greater);
assert_eq!(version_cmp(b"apple", b"ant"), Ordering::Greater);
assert_eq!(version_cmp("ant", "apple"), Ordering::Less);
assert_eq!(version_cmp(b"ant", b"apple"), Ordering::Less);
// Uppercase letters
assert_eq!(
version_cmp("Beef", "apple"),
version_cmp(b"Beef", b"apple"),
Ordering::Less,
"Uppercase letters are sorted before all lowercase letters"
);
assert_eq!(version_cmp("Apple", "apple"), Ordering::Less);
assert_eq!(version_cmp(b"Apple", b"apple"), Ordering::Less);
assert_eq!(version_cmp("apple", "aPple"), Ordering::Greater);
assert_eq!(version_cmp(b"apple", b"aPple"), Ordering::Greater);
// Numbers
assert_eq!(
version_cmp("100", "20"),
version_cmp(b"100", b"20"),
Ordering::Greater,
"Greater numbers are greater even if they start with a smaller digit",
);
assert_eq!(
version_cmp("20", "20"),
version_cmp(b"20", b"20"),
Ordering::Equal,
"Equal numbers are equal"
);
assert_eq!(
version_cmp("15", "200"),
version_cmp(b"15", b"200"),
Ordering::Less,
"Small numbers are smaller"
);
// Comparing numbers with other characters
assert_eq!(
version_cmp("1000", "apple"),
version_cmp(b"1000", b"apple"),
Ordering::Less,
"Numbers are sorted before other characters"
);
assert_eq!(
// spell-checker:disable-next-line
version_cmp("file1000", "fileapple"),
version_cmp(b"file1000", b"fileapple"),
Ordering::Less,
"Numbers in the middle of the name are sorted before other characters"
);
// Leading zeroes
assert_eq!(
version_cmp("012", "12"),
version_cmp(b"012", b"12"),
Ordering::Equal,
"A single leading zero does not make a difference"
);
assert_eq!(
version_cmp("000800", "0000800"),
version_cmp(b"000800", b"0000800"),
Ordering::Equal,
"Multiple leading zeros do not make a difference"
);
// Numbers and other characters combined
assert_eq!(version_cmp("ab10", "aa11"), Ordering::Greater);
assert_eq!(version_cmp(b"ab10", b"aa11"), Ordering::Greater);
assert_eq!(
version_cmp("aa10", "aa11"),
version_cmp(b"aa10", b"aa11"),
Ordering::Less,
"Numbers after other characters are handled correctly."
);
assert_eq!(
version_cmp("aa2", "aa100"),
version_cmp(b"aa2", b"aa100"),
Ordering::Less,
"Numbers after alphabetical characters are handled correctly."
);
assert_eq!(
version_cmp("aa10bb", "aa11aa"),
version_cmp(b"aa10bb", b"aa11aa"),
Ordering::Less,
"Number is used even if alphabetical characters after it differ."
);
assert_eq!(
version_cmp("aa10aa0010", "aa11aa1"),
version_cmp(b"aa10aa0010", b"aa11aa1"),
Ordering::Less,
"Second number is ignored if the first number differs."
);
assert_eq!(
version_cmp("aa10aa0010", "aa10aa1"),
version_cmp(b"aa10aa0010", b"aa10aa1"),
Ordering::Greater,
"Second number is used if the rest is equal."
);
assert_eq!(
version_cmp("aa10aa0010", "aa00010aa1"),
version_cmp(b"aa10aa0010", b"aa00010aa1"),
Ordering::Greater,
"Second number is used if the rest is equal up to leading zeroes of the first number."
);
assert_eq!(
version_cmp("aa10aa0022", "aa010aa022"),
version_cmp(b"aa10aa0022", b"aa010aa022"),
Ordering::Equal,
"Test multiple numeric values with leading zeros"
);
assert_eq!(
version_cmp("file-1.4", "file-1.13"),
version_cmp(b"file-1.4", b"file-1.13"),
Ordering::Less,
"Periods are handled as normal text, not as a decimal point."
);
@@ -299,42 +305,48 @@ mod tests {
// u64 == 18446744073709551615 so this should be plenty:
// 20000000000000000000000
assert_eq!(
version_cmp("aa2000000000000000000000bb", "aa002000000000000000000001bb"),
version_cmp(
b"aa2000000000000000000000bb",
b"aa002000000000000000000001bb"
),
Ordering::Less,
"Numbers larger than u64::MAX are handled correctly without crashing"
);
assert_eq!(
version_cmp("aa2000000000000000000000bb", "aa002000000000000000000000bb"),
version_cmp(
b"aa2000000000000000000000bb",
b"aa002000000000000000000000bb"
),
Ordering::Equal,
"Leading zeroes for numbers larger than u64::MAX are \
handled correctly without crashing"
);
assert_eq!(
version_cmp(" a", "a"),
version_cmp(b" a", b"a"),
Ordering::Greater,
"Whitespace is after letters because letters are before non-letters"
);
assert_eq!(
version_cmp("a~", "ab"),
version_cmp(b"a~", b"ab"),
Ordering::Less,
"A tilde is before other letters"
);
assert_eq!(
version_cmp("a~", "a"),
version_cmp(b"a~", b"a"),
Ordering::Less,
"A tilde is before the line end"
);
assert_eq!(
version_cmp("~", ""),
version_cmp(b"~", b""),
Ordering::Greater,
"A tilde is after the empty string"
);
assert_eq!(
version_cmp(".f", ".1"),
version_cmp(b".f", b".1"),
Ordering::Greater,
"if both start with a dot it is ignored for the comparison"
);
@@ -342,17 +354,17 @@ mod tests {
// The following tests are incompatible with GNU as of 2021/06.
// I think that's because of a bug in GNU, reported as https://lists.gnu.org/archive/html/bug-coreutils/2021-06/msg00045.html
assert_eq!(
version_cmp("a..a", "a.+"),
version_cmp(b"a..a", b"a.+"),
Ordering::Less,
".a is stripped before the comparison"
);
assert_eq!(
version_cmp("a.", "a+"),
version_cmp(b"a.", b"a+"),
Ordering::Greater,
". is not stripped before the comparison"
);
assert_eq!(
version_cmp("a\0a", "a"),
version_cmp(b"a\0a", b"a"),
Ordering::Greater,
"NULL bytes are handled comparison"
);
tests/fixtures/sort/keys_closed_range.expected.debug
Vendored
+7 -7
View File
@@ -8,11 +8,11 @@ aa bb cc
_
________
èè éé èè
_
________
πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬
__
________
πŸ’£πŸ’£ πŸ’£πŸ’£ πŸ’£πŸ’£
__
_
______________
πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬
_
___________________________________
πŸ’£πŸ’£ πŸ’£πŸ’£ πŸ’£πŸ’£
_
__________________________
tests/fixtures/sort/keys_multiple_ranges.expected.debug
Vendored
+8 -8
View File
@@ -11,14 +11,14 @@ aa bb cc
___
________
èè éé èè
___
___
________
πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬
___
___
________
πŸ’£πŸ’£ πŸ’£πŸ’£ πŸ’£πŸ’£
_____
_____
______________
πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬
____________
____________
___________________________________
πŸ’£πŸ’£ πŸ’£πŸ’£ πŸ’£πŸ’£
_________
_________
__________________________
tests/fixtures/sort/keys_no_field_match.expected.debug
Vendored
+6 -6
View File
@@ -8,11 +8,11 @@ gg aa cc
^ no match for key
________
èè éé èè
^ no match for key
________
πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬
^ no match for key
________
πŸ’£πŸ’£ πŸ’£πŸ’£ πŸ’£πŸ’£
^ no match for key
______________
πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬
^ no match for key
___________________________________
πŸ’£πŸ’£ πŸ’£πŸ’£ πŸ’£πŸ’£
^ no match for key
__________________________
tests/fixtures/sort/keys_open_ended.expected
Vendored
+1 -1
View File
@@ -1,6 +1,6 @@
gg aa cc
dd aa ff
aa bb cc
èè éé èè
πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬
πŸ’£πŸ’£ πŸ’£πŸ’£ πŸ’£πŸ’£
èè éé èè
tests/fixtures/sort/keys_open_ended.expected.debug
Vendored
+6 -6
View File
@@ -7,12 +7,12 @@ ________
aa bb cc
____
________
èè éé èè
____
________
πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬ πŸ‘©β€πŸ”¬
_____
________
______________________
___________________________________
πŸ’£πŸ’£ πŸ’£πŸ’£ πŸ’£πŸ’£
_______
________________
__________________________
èè éé èè
________
______________
tests/test_util_name.rs
+2
View File
@@ -46,6 +46,7 @@ fn execution_phrase_double() {
}
#[test]
#[ignore = "Test assumes error upon non-UTF8"]
#[cfg(feature = "sort")]
fn util_name_double() {
use std::{
@@ -71,6 +72,7 @@ fn util_name_double() {
}
#[test]
#[ignore = "Test assumes error upon non-UTF8"]
#[cfg(feature = "sort")]
#[cfg(unix)]
fn util_name_single() {