mirror of
https://github.com/uutils/coreutils.git
synced 2026-06-10 15:48:22 -07:00
fix(wc):GNU wc-cpu.sh (#9144)
* wc: align SIMD policy integration - use SimdPolicy::detect with hardware feature labeling - keep SIMD behavior respecting GLIBC_TUNABLES - consolidate wc SIMD debug output and tests * refactor(wc): simplify SIMD feature collection in debug output - Changed multi-line SIMD feature vector creation to a single-line expression for improved readability and consistency with surrounding code. - No functional changes; only stylistic refactoring in the wc debug logic. * feat(wc): enhance debug output for SIMD hardware support limitations Add new localization strings and logic to provide detailed debug information when SIMD support is limited by GLIBC_TUNABLES, including lists of disabled and enabled features. Refactor SIMD allowance check for better accuracy in detecting runtime support. * refactor: consolidate SIMD feature handling in wc command Refactor SIMD feature detection and reporting in the wc utility by introducing a WcSimdFeatures struct to group enabled, disabled, and runtime-disabled features. This replaces multiple separate functions with a single function, improving code organization and efficiency by reducing redundant iterations over feature lists. Also rename helper functions for clarity and update debug output logic accordingly. * Update src/uu/wc/src/wc.rs Co-authored-by: Dorian Péron <72708393+RenjiSann@users.noreply.github.com> * Update src/uu/wc/locales/en-US.ftl Co-authored-by: Dorian Péron <72708393+RenjiSann@users.noreply.github.com> * Update src/uu/wc/locales/fr-FR.ftl Co-authored-by: Dorian Péron <72708393+RenjiSann@users.noreply.github.com> * feat(wc): import show_error for enhanced error reporting Add the show_error import from uucore to enable better error handling in the wc utility, allowing for consistent error messages in line with the project's style. --------- Co-authored-by: Dorian Péron <72708393+RenjiSann@users.noreply.github.com>
This commit is contained in:
@@ -1,4 +1,6 @@
|
||||
AFAICT
|
||||
asimd
|
||||
ASIMD
|
||||
alloc
|
||||
arity
|
||||
autogenerate
|
||||
@@ -71,6 +73,7 @@ hardlink
|
||||
hardlinks
|
||||
hasher
|
||||
hashsums
|
||||
hwcaps
|
||||
infile
|
||||
iflag
|
||||
iflags
|
||||
@@ -150,6 +153,8 @@ tokenize
|
||||
toolchain
|
||||
totalram
|
||||
truthy
|
||||
tunables
|
||||
TUNABLES
|
||||
ucase
|
||||
unbuffered
|
||||
udeps
|
||||
|
||||
+10
-5
@@ -18,16 +18,21 @@ workspace = true
|
||||
path = "src/wc.rs"
|
||||
|
||||
[dependencies]
|
||||
clap = { workspace = true }
|
||||
uucore = { workspace = true, features = ["parser", "pipes", "quoting-style"] }
|
||||
bytecount = { workspace = true, features = ["runtime-dispatch-simd"] }
|
||||
thiserror = { workspace = true }
|
||||
unicode-width = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
fluent = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
uucore = { workspace = true, features = [
|
||||
"hardware",
|
||||
"parser",
|
||||
"pipes",
|
||||
"quoting-style",
|
||||
] }
|
||||
unicode-width = { workspace = true }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
nix = { workspace = true }
|
||||
libc = { workspace = true }
|
||||
nix = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
divan = { workspace = true }
|
||||
|
||||
@@ -31,3 +31,10 @@ decoder-error-io = underlying bytestream error: { $error }
|
||||
# Other messages
|
||||
wc-standard-input = standard input
|
||||
wc-total = total
|
||||
|
||||
# Debug messages
|
||||
wc-debug-hw-unavailable = debug: hardware support unavailable on this CPU
|
||||
wc-debug-hw-using = debug: using hardware support (features: { $features })
|
||||
wc-debug-hw-disabled-env = debug: hardware support disabled by environment
|
||||
wc-debug-hw-disabled-glibc = debug: hardware support disabled by GLIBC_TUNABLES ({ $features })
|
||||
wc-debug-hw-limited-glibc = debug: hardware support limited by GLIBC_TUNABLES (disabled: { $disabled }; enabled: { $enabled })
|
||||
|
||||
@@ -31,3 +31,10 @@ decoder-error-io = erreur du flux d'octets sous-jacent : { $error }
|
||||
# Autres messages
|
||||
wc-standard-input = entrée standard
|
||||
wc-total = total
|
||||
|
||||
# Messages de débogage
|
||||
wc-debug-hw-unavailable = debug : prise en charge matérielle indisponible sur ce CPU
|
||||
wc-debug-hw-using = debug : utilisation de l'accélération matérielle (fonctions : { $features })
|
||||
wc-debug-hw-disabled-env = debug : prise en charge matérielle désactivée par l'environnement
|
||||
wc-debug-hw-disabled-glibc = debug : prise en charge matérielle désactivée par GLIBC_TUNABLES ({ $features })
|
||||
wc-debug-hw-limited-glibc = debug : prise en charge matérielle limitée par GLIBC_TUNABLES (désactivé : { $disabled } ; activé : { $enabled })
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
// file that was distributed with this source code.
|
||||
|
||||
// cSpell:ignore sysconf
|
||||
use crate::word_count::WordCount;
|
||||
use crate::{wc_simd_allowed, word_count::WordCount};
|
||||
use uucore::hardware::SimdPolicy;
|
||||
|
||||
use super::WordCountable;
|
||||
|
||||
@@ -232,6 +233,8 @@ pub(crate) fn count_bytes_chars_and_lines_fast<
|
||||
) -> (WordCount, Option<io::Error>) {
|
||||
let mut total = WordCount::default();
|
||||
let buf: &mut [u8] = &mut AlignedBuffer::default().data;
|
||||
let policy = SimdPolicy::detect();
|
||||
let simd_allowed = wc_simd_allowed(policy);
|
||||
loop {
|
||||
match handle.read(buf) {
|
||||
Ok(0) => return (total, None),
|
||||
@@ -240,10 +243,18 @@ pub(crate) fn count_bytes_chars_and_lines_fast<
|
||||
total.bytes += n;
|
||||
}
|
||||
if COUNT_CHARS {
|
||||
total.chars += bytecount::num_chars(&buf[..n]);
|
||||
total.chars += if simd_allowed {
|
||||
bytecount::num_chars(&buf[..n])
|
||||
} else {
|
||||
bytecount::naive_num_chars(&buf[..n])
|
||||
};
|
||||
}
|
||||
if COUNT_LINES {
|
||||
total.lines += bytecount::count(&buf[..n], b'\n');
|
||||
total.lines += if simd_allowed {
|
||||
bytecount::count(&buf[..n], b'\n')
|
||||
} else {
|
||||
bytecount::naive_count(&buf[..n], b'\n')
|
||||
};
|
||||
}
|
||||
}
|
||||
Err(ref e) if e.kind() == ErrorKind::Interrupted => (),
|
||||
|
||||
+126
-1
@@ -29,9 +29,10 @@ use uucore::translate;
|
||||
use uucore::{
|
||||
error::{FromIo, UError, UResult},
|
||||
format_usage,
|
||||
hardware::{HardwareFeature, HasHardwareFeatures as _, SimdPolicy},
|
||||
parser::shortcut_value_parser::ShortcutValueParser,
|
||||
quoting_style::{self, QuotingStyle},
|
||||
show,
|
||||
show, show_error,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
@@ -49,6 +50,7 @@ struct Settings<'a> {
|
||||
show_lines: bool,
|
||||
show_words: bool,
|
||||
show_max_line_length: bool,
|
||||
debug: bool,
|
||||
files0_from: Option<Input<'a>>,
|
||||
total_when: TotalWhen,
|
||||
}
|
||||
@@ -62,6 +64,7 @@ impl Default for Settings<'_> {
|
||||
show_lines: true,
|
||||
show_words: true,
|
||||
show_max_line_length: false,
|
||||
debug: false,
|
||||
files0_from: None,
|
||||
total_when: TotalWhen::default(),
|
||||
}
|
||||
@@ -85,6 +88,7 @@ impl<'a> Settings<'a> {
|
||||
show_lines: matches.get_flag(options::LINES),
|
||||
show_words: matches.get_flag(options::WORDS),
|
||||
show_max_line_length: matches.get_flag(options::MAX_LINE_LENGTH),
|
||||
debug: matches.get_flag(options::DEBUG),
|
||||
files0_from,
|
||||
total_when,
|
||||
};
|
||||
@@ -95,6 +99,7 @@ impl<'a> Settings<'a> {
|
||||
Self {
|
||||
files0_from: settings.files0_from,
|
||||
total_when,
|
||||
debug: settings.debug,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -122,6 +127,7 @@ mod options {
|
||||
pub static MAX_LINE_LENGTH: &str = "max-line-length";
|
||||
pub static TOTAL: &str = "total";
|
||||
pub static WORDS: &str = "words";
|
||||
pub static DEBUG: &str = "debug";
|
||||
}
|
||||
static ARG_FILES: &str = "files";
|
||||
static STDIN_REPR: &str = "-";
|
||||
@@ -445,6 +451,12 @@ pub fn uu_app() -> Command {
|
||||
.help(translate!("wc-help-words"))
|
||||
.action(ArgAction::SetTrue),
|
||||
)
|
||||
.arg(
|
||||
Arg::new(options::DEBUG)
|
||||
.long(options::DEBUG)
|
||||
.action(ArgAction::SetTrue)
|
||||
.hide(true),
|
||||
)
|
||||
.arg(
|
||||
Arg::new(ARG_FILES)
|
||||
.action(ArgAction::Append)
|
||||
@@ -805,6 +817,74 @@ fn escape_name_wrapper(name: &OsStr) -> String {
|
||||
.expect("All escaped names with the escaping option return valid strings.")
|
||||
}
|
||||
|
||||
fn hardware_feature_label(feature: HardwareFeature) -> &'static str {
|
||||
match feature {
|
||||
HardwareFeature::Avx512 => "AVX512F",
|
||||
HardwareFeature::Avx2 => "AVX2",
|
||||
HardwareFeature::PclMul => "PCLMUL",
|
||||
HardwareFeature::Vmull => "VMULL",
|
||||
HardwareFeature::Sse2 => "SSE2",
|
||||
HardwareFeature::Asimd => "ASIMD",
|
||||
}
|
||||
}
|
||||
|
||||
fn is_simd_runtime_feature(feature: &HardwareFeature) -> bool {
|
||||
matches!(
|
||||
feature,
|
||||
HardwareFeature::Avx2 | HardwareFeature::Sse2 | HardwareFeature::Asimd
|
||||
)
|
||||
}
|
||||
|
||||
fn is_simd_debug_feature(feature: &HardwareFeature) -> bool {
|
||||
matches!(
|
||||
feature,
|
||||
HardwareFeature::Avx512
|
||||
| HardwareFeature::Avx2
|
||||
| HardwareFeature::Sse2
|
||||
| HardwareFeature::Asimd
|
||||
)
|
||||
}
|
||||
|
||||
struct WcSimdFeatures {
|
||||
enabled: Vec<HardwareFeature>,
|
||||
disabled: Vec<HardwareFeature>,
|
||||
disabled_runtime: Vec<HardwareFeature>,
|
||||
}
|
||||
|
||||
fn wc_simd_features(policy: &SimdPolicy) -> WcSimdFeatures {
|
||||
let enabled = policy
|
||||
.iter_features()
|
||||
.filter(is_simd_runtime_feature)
|
||||
.collect();
|
||||
|
||||
let mut disabled = Vec::new();
|
||||
let mut disabled_runtime = Vec::new();
|
||||
for feature in policy.disabled_features() {
|
||||
if is_simd_debug_feature(&feature) {
|
||||
disabled.push(feature);
|
||||
}
|
||||
if is_simd_runtime_feature(&feature) {
|
||||
disabled_runtime.push(feature);
|
||||
}
|
||||
}
|
||||
|
||||
WcSimdFeatures {
|
||||
enabled,
|
||||
disabled,
|
||||
disabled_runtime,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn wc_simd_allowed(policy: &SimdPolicy) -> bool {
|
||||
let disabled_features = policy.disabled_features();
|
||||
if disabled_features.iter().any(is_simd_runtime_feature) {
|
||||
return false;
|
||||
}
|
||||
policy
|
||||
.iter_features()
|
||||
.any(|feature| is_simd_runtime_feature(&feature))
|
||||
}
|
||||
|
||||
fn wc(inputs: &Inputs, settings: &Settings) -> UResult<()> {
|
||||
let mut total_word_count = WordCount::default();
|
||||
let mut num_inputs: usize = 0;
|
||||
@@ -814,6 +894,51 @@ fn wc(inputs: &Inputs, settings: &Settings) -> UResult<()> {
|
||||
_ => (compute_number_width(inputs, settings), true),
|
||||
};
|
||||
|
||||
if settings.debug {
|
||||
let policy = SimdPolicy::detect();
|
||||
let features = wc_simd_features(policy);
|
||||
|
||||
let enabled: Vec<&'static str> = features
|
||||
.enabled
|
||||
.iter()
|
||||
.copied()
|
||||
.map(hardware_feature_label)
|
||||
.collect();
|
||||
let disabled: Vec<&'static str> = features
|
||||
.disabled
|
||||
.iter()
|
||||
.copied()
|
||||
.map(hardware_feature_label)
|
||||
.collect();
|
||||
|
||||
let enabled_empty = enabled.is_empty();
|
||||
let disabled_empty = disabled.is_empty();
|
||||
let runtime_disabled = !features.disabled_runtime.is_empty();
|
||||
|
||||
if enabled_empty && !runtime_disabled {
|
||||
show_error!("{}", translate!("wc-debug-hw-unavailable"));
|
||||
} else if runtime_disabled {
|
||||
show_error!(
|
||||
"{}",
|
||||
translate!("wc-debug-hw-disabled-glibc", "features" => disabled.join(", "))
|
||||
);
|
||||
} else if !enabled_empty && disabled_empty {
|
||||
show_error!(
|
||||
"{}",
|
||||
translate!("wc-debug-hw-using", "features" => enabled.join(", "))
|
||||
);
|
||||
} else {
|
||||
show_error!(
|
||||
"{}",
|
||||
translate!(
|
||||
"wc-debug-hw-limited-glibc",
|
||||
"disabled" => disabled.join(", "),
|
||||
"enabled" => enabled.join(", ")
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for maybe_input in inputs.try_iter(settings)? {
|
||||
num_inputs += 1;
|
||||
|
||||
|
||||
@@ -214,8 +214,9 @@ impl SimdPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if any SIMD feature remains enabled after applying GLIBC_TUNABLES.
|
||||
pub fn allows_simd(&self) -> bool {
|
||||
self.disabled_by_env.is_empty()
|
||||
self.iter_features().next().is_some()
|
||||
}
|
||||
|
||||
pub fn disabled_features(&self) -> Vec<HardwareFeature> {
|
||||
|
||||
@@ -808,3 +808,69 @@ fn wc_w_words_with_emoji_separator() {
|
||||
.succeeds()
|
||||
.stdout_contains("3");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn test_simd_respects_glibc_tunables() {
|
||||
// Ensure debug output reflects that SIMD paths are disabled via GLIBC_TUNABLES
|
||||
let debug_output = new_ucmd!()
|
||||
.args(&["-l", "--debug", "/dev/null"])
|
||||
.env("GLIBC_TUNABLES", "glibc.cpu.hwcaps=-AVX2,-AVX512F")
|
||||
.succeeds()
|
||||
.stderr_str()
|
||||
.to_string();
|
||||
assert!(
|
||||
!debug_output.contains("using hardware support"),
|
||||
"SIMD should be reported as disabled when GLIBC_TUNABLES blocks AVX features: {debug_output}"
|
||||
);
|
||||
assert!(
|
||||
debug_output.contains("hardware support disabled"),
|
||||
"Debug output should acknowledge GLIBC_TUNABLES restrictions: {debug_output}"
|
||||
);
|
||||
|
||||
// WC results should be identical with and without GLIBC_TUNABLES overrides
|
||||
let sample_sizes = [0usize, 1, 7, 128, 513, 999];
|
||||
use std::fmt::Write as _;
|
||||
for &lines in &sample_sizes {
|
||||
let content: String = (0..lines).fold(String::new(), |mut acc, i| {
|
||||
// Build the input buffer efficiently without allocating per line.
|
||||
let _ = writeln!(acc, "{i}");
|
||||
acc
|
||||
});
|
||||
|
||||
let base = new_ucmd!()
|
||||
.arg("-l")
|
||||
.pipe_in(content.clone())
|
||||
.succeeds()
|
||||
.stdout_str()
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
let no_avx512 = new_ucmd!()
|
||||
.arg("-l")
|
||||
.env("GLIBC_TUNABLES", "glibc.cpu.hwcaps=-AVX512F")
|
||||
.pipe_in(content.clone())
|
||||
.succeeds()
|
||||
.stdout_str()
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
let no_avx2_avx512 = new_ucmd!()
|
||||
.arg("-l")
|
||||
.env("GLIBC_TUNABLES", "glibc.cpu.hwcaps=-AVX2,-AVX512F")
|
||||
.pipe_in(content)
|
||||
.succeeds()
|
||||
.stdout_str()
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
assert_eq!(
|
||||
base, no_avx512,
|
||||
"Line counts should not change when AVX512 is disabled (lines={lines})"
|
||||
);
|
||||
assert_eq!(
|
||||
base, no_avx2_avx512,
|
||||
"Line counts should not change when AVX2/AVX512 are disabled (lines={lines})"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user