diff --git a/src/uu/sort/Cargo.toml b/src/uu/sort/Cargo.toml index 476375516..8b422898a 100644 --- a/src/uu/sort/Cargo.toml +++ b/src/uu/sort/Cargo.toml @@ -19,6 +19,9 @@ workspace = true [lib] path = "src/sort.rs" +[features] +i18n-collator = ["uucore/i18n-collator"] + [dependencies] bigdecimal = { workspace = true } binary-heap-plus = { workspace = true } @@ -39,6 +42,7 @@ uucore = { workspace = true, features = [ "parser-size", "version-cmp", "i18n-decimal", + "i18n-collator", ] } fluent = { workspace = true } diff --git a/src/uu/sort/src/sort.rs b/src/uu/sort/src/sort.rs index ddbf22576..efce29180 100644 --- a/src/uu/sort/src/sort.rs +++ b/src/uu/sort/src/sort.rs @@ -23,6 +23,7 @@ use chunks::LineData; use clap::builder::ValueParser; use clap::{Arg, ArgAction, ArgMatches, Command}; use custom_str_cmp::custom_str_cmp; + use ext_sort::ext_sort; use fnv::FnvHasher; use numeric_str_cmp::{NumInfo, NumInfoParseSettings, human_numeric_str_cmp, numeric_str_cmp}; @@ -47,6 +48,8 @@ use uucore::error::{FromIo, strip_errno}; use uucore::error::{UError, UResult, USimpleError, UUsageError}; use uucore::extendedbigdecimal::ExtendedBigDecimal; use uucore::format_usage; +#[cfg(feature = "i18n-collator")] +use uucore::i18n::collator::locale_cmp; use uucore::i18n::decimal::locale_decimal_separator; use uucore::line_ending::LineEnding; use uucore::parser::num_parser::{ExtendedParser, ExtendedParserError}; @@ -318,7 +321,10 @@ impl GlobalSettings { /// Precompute some data needed for sorting. /// This function **must** be called before starting to sort, and `GlobalSettings` may not be altered /// afterwards. - fn init_precomputed(&mut self) { + /// + /// When i18n-collator is enabled, `disable_fast_lexicographic` should be set to true if we're + /// in a UTF-8 locale (to force locale-aware collation instead of byte comparison). + fn init_precomputed(&mut self, disable_fast_lexicographic: bool) { self.precomputed.needs_tokens = self.selectors.iter().any(|s| s.needs_tokens); self.precomputed.selections_per_line = self.selectors.iter().filter(|s| s.needs_selection).count(); @@ -333,11 +339,15 @@ impl GlobalSettings { .filter(|s| matches!(s.settings.mode, SortMode::GeneralNumeric)) .count(); - self.precomputed.fast_lexicographic = self.can_use_fast_lexicographic(); + self.precomputed.fast_lexicographic = + !disable_fast_lexicographic && self.can_use_fast_lexicographic(); self.precomputed.fast_ascii_insensitive = self.can_use_fast_ascii_insensitive(); } /// Returns true when the fast lexicographic path can be used safely. + /// Note: When i18n-collator is enabled, the caller must have already determined + /// whether locale-aware collation is needed (via checking if we're in a UTF-8 locale). + /// This check is performed in uumain() before init_precomputed() is called. fn can_use_fast_lexicographic(&self) -> bool { self.mode == SortMode::Default && !self.ignore_case @@ -2065,7 +2075,15 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { emit_debug_warnings(&settings, &global_flags, &legacy_warnings); } - settings.init_precomputed(); + // Initialize locale collation if needed (UTF-8 locales) + // This MUST happen before init_precomputed() to avoid the performance regression + #[cfg(feature = "i18n-collator")] + let needs_locale_collation = uucore::i18n::collator::init_locale_collation(); + + #[cfg(not(feature = "i18n-collator"))] + let needs_locale_collation = false; + + settings.init_precomputed(needs_locale_collation); let result = exec(&mut files, &settings, output, &mut tmp_dir); // Wait here if `SIGINT` was received, @@ -2446,13 +2464,36 @@ fn compare_by<'a>( } SortMode::Month => month_compare(a_str, b_str), SortMode::Version => version_cmp(a_str, b_str), - SortMode::Default => custom_str_cmp( - a_str, - b_str, - settings.ignore_non_printing, - settings.dictionary_order, - settings.ignore_case, - ), + SortMode::Default => { + // Use locale-aware comparison if feature is enabled and no custom flags are set + #[cfg(feature = "i18n-collator")] + { + if settings.ignore_case + || settings.dictionary_order + || settings.ignore_non_printing + { + custom_str_cmp( + a_str, + b_str, + settings.ignore_non_printing, + settings.dictionary_order, + settings.ignore_case, + ) + } else { + locale_cmp(a_str, b_str) + } + } + #[cfg(not(feature = "i18n-collator"))] + { + custom_str_cmp( + a_str, + b_str, + settings.ignore_non_printing, + settings.dictionary_order, + settings.ignore_case, + ) + } + } }; if cmp != Ordering::Equal { return if settings.reverse { cmp.reverse() } else { cmp }; diff --git a/src/uucore/src/lib/features/i18n/collator.rs b/src/uucore/src/lib/features/i18n/collator.rs index fda8cd6e0..f0a9e6b35 100644 --- a/src/uucore/src/lib/features/i18n/collator.rs +++ b/src/uucore/src/lib/features/i18n/collator.rs @@ -30,6 +30,45 @@ pub fn init_collator(opts: CollatorOptions) { .expect("Collator already initialized"); } +/// Initialize the collator for locale-aware string comparison if needed. +/// +/// This function checks if the current locale requires locale-aware collation +/// (UTF-8 encoding) and initializes the ICU collator with appropriate settings +/// if necessary. For C/POSIX locales, no initialization is needed as byte +/// comparison is sufficient. +/// +/// # Returns +/// +/// `true` if the collator was initialized for a UTF-8 locale, `false` if +/// using C/POSIX locale (no initialization needed). +/// +/// # Example +/// +/// ``` +/// use uucore::i18n::collator::init_locale_collation; +/// +/// if init_locale_collation() { +/// // Using locale-aware collation +/// } else { +/// // Using byte comparison (C/POSIX locale) +/// } +/// ``` +pub fn init_locale_collation() -> bool { + use crate::i18n::{UEncoding, get_locale_encoding}; + + // Check if we need locale-aware collation + if get_locale_encoding() != UEncoding::Utf8 { + // C/POSIX locale - no collator needed + return false; + } + + // UTF-8 locale - initialize collator with Shifted mode to match GNU behavior + let mut opts = CollatorOptions::default(); + opts.alternate_handling = Some(AlternateHandling::Shifted); + + try_init_collator(opts) +} + /// Compare both strings with regard to the current locale. pub fn locale_cmp(left: &[u8], right: &[u8]) -> Ordering { // If the detected locale is 'C', just do byte-wise comparison diff --git a/src/uucore/src/lib/features/i18n/mod.rs b/src/uucore/src/lib/features/i18n/mod.rs index d47f2df98..79c804a03 100644 --- a/src/uucore/src/lib/features/i18n/mod.rs +++ b/src/uucore/src/lib/features/i18n/mod.rs @@ -20,7 +20,9 @@ pub enum UEncoding { Utf8, } -const DEFAULT_LOCALE: Locale = locale!("en-US-posix"); +// Use "und" (undefined) as the marker for C/POSIX locale +// This ensures real locales like "en-US" won't match +const DEFAULT_LOCALE: Locale = locale!("und"); /// Look at 3 environment variables in the following order /// @@ -38,6 +40,11 @@ fn get_locale_from_env(locale_name: &str) -> (Locale, UEncoding) { let mut split = locale_var_str.split(&['.', '@']); if let Some(simple) = split.next() { + // Handle explicit C and POSIX locales - these should always use byte comparison + if simple == "C" || simple == "POSIX" { + return (DEFAULT_LOCALE, UEncoding::Ascii); + } + // Naively convert the locale name to BCP47 tag format. // // See https://en.wikipedia.org/wiki/IETF_language_tag diff --git a/tests/by-util/test_sort.rs b/tests/by-util/test_sort.rs index b478912dd..0106d719f 100644 --- a/tests/by-util/test_sort.rs +++ b/tests/by-util/test_sort.rs @@ -2463,4 +2463,58 @@ fn test_start_buffer() { .stdout_only_bytes(&expected); } +#[test] +fn test_locale_collation_c_locale() { + // C locale uses byte order - this is deterministic and tests the fix for #9148 + // Accented characters (UTF-8 multibyte) sort after ASCII letters + let input = "é\ne\nE\na\nA\nz\n"; + // C locale byte order: A=0x41, E=0x45, a=0x61, e=0x65, z=0x7A, é=0xC3 0xA9 + let expected = "A\nE\na\ne\nz\né\n"; + + new_ucmd!() + .env("LC_ALL", "C") + .pipe_in(input) + .succeeds() + .stdout_is(expected); +} + +#[test] +fn test_locale_collation_utf8() { + // Test French UTF-8 locale handling - behavior depends on i18n-collator feature + // With feature: locale-aware collation (é sorts near e) + // Without feature: byte order (é after z, since 0xC3A9 > 0x7A) + let input = "z\né\ne\na\n"; + + let result = new_ucmd!() + .env("LC_ALL", "fr_FR.UTF-8") + .pipe_in(input) + .succeeds(); + + let output = result.stdout_str(); + let lines: Vec<&str> = output.lines().collect(); + + assert_eq!(lines.len(), 4, "Expected 4 sorted lines"); + assert_eq!(lines[0], "a", "'a' (0x61) should always sort first"); + + // Validate based on which collation mode is active + if lines[3] == "é" { + // Byte order mode: é (0xC3A9) > z (0x7A) + assert_eq!( + lines, + vec!["a", "e", "z", "é"], + "Byte order mode: expected a < e < z < é" + ); + } else { + // Locale collation mode: é sorts with base letter e + assert_eq!(lines[3], "z", "Locale mode: 'z' should sort last"); + let z_pos = lines.iter().position(|&x| x == "z").unwrap(); + let e_pos = lines.iter().position(|&x| x == "e").unwrap(); + let e_accent_pos = lines.iter().position(|&x| x == "é").unwrap(); + assert!( + e_pos < z_pos && e_accent_pos < z_pos, + "Locale mode: 'e' ({e_pos}) and 'é' ({e_accent_pos}) should sort before 'z' ({z_pos})" + ); + } +} + /* spell-checker: enable */