From 55e756f67a2b33e5d950005cdde36852a5eb3dde Mon Sep 17 00:00:00 2001 From: Jan Verbeek Date: Wed, 26 Mar 2025 15:51:55 +0100 Subject: [PATCH] shuf: feature: Add --random-seed option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a new option to get reproducible output from a seed. This was already possible with --random-source, but doing that properly was tricky and had poor performance. Adding this option implies a commitment to keep using the exact same algorithms in the future. For that reason we only use third-party libraries for well-known algorithms and implement our own distributions on top of that. ----- As a teenager on King's Day I once used `shuf` for divination. People paid €0.50 to enter a cramped tent and sat down next to me behind an old netbook. I would ask their name and their sun sign and pipe this information into `shuf --random-source=/dev/stdin`, which selected pseudo-random dictionary words and `tee`d them into `espeak`. If someone's name was too short `shuf` crashed with an end of file error. --random-seed would have worked better. --- .../cspell.dictionaries/jargon.wordlist.txt | 1 + .../cspell.dictionaries/people.wordlist.txt | 3 + Cargo.lock | 2 + Cargo.toml | 1 + src/uu/shuf/Cargo.toml | 2 + src/uu/shuf/locales/en-US.ftl | 1 + src/uu/shuf/src/compat_random_source.rs | 24 +++- src/uu/shuf/src/random_seed.rs | 118 ++++++++++++++++++ src/uu/shuf/src/shuf.rs | 77 ++++++++---- tests/by-util/test_shuf.rs | 49 ++++++++ 10 files changed, 250 insertions(+), 28 deletions(-) create mode 100644 src/uu/shuf/src/random_seed.rs diff --git a/.vscode/cspell.dictionaries/jargon.wordlist.txt b/.vscode/cspell.dictionaries/jargon.wordlist.txt index 2ca152125..7ba13ab80 100644 --- a/.vscode/cspell.dictionaries/jargon.wordlist.txt +++ b/.vscode/cspell.dictionaries/jargon.wordlist.txt @@ -55,6 +55,7 @@ fileio filesystem filesystems flamegraph +footgun freeram fsxattr fullblock diff --git a/.vscode/cspell.dictionaries/people.wordlist.txt b/.vscode/cspell.dictionaries/people.wordlist.txt index 8fe38d885..446c00df4 100644 --- a/.vscode/cspell.dictionaries/people.wordlist.txt +++ b/.vscode/cspell.dictionaries/people.wordlist.txt @@ -37,6 +37,9 @@ Boden Garman Chirag B Jadwani Chirag Jadwani +Daniel Lemire + Daniel + Lemire Derek Chiang Derek Chiang diff --git a/Cargo.lock b/Cargo.lock index c5e5d61dd..591b5c558 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3869,7 +3869,9 @@ dependencies = [ "fluent", "itoa", "rand 0.9.2", + "rand_chacha 0.9.0", "rand_core 0.9.5", + "sha3", "uucore", ] diff --git a/Cargo.toml b/Cargo.toml index d3eee72cc..de95e8fa8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -356,6 +356,7 @@ phf_codegen = "0.13.1" platform-info = "2.0.3" procfs = "0.18" rand = { version = "0.9.0", features = ["small_rng"] } +rand_chacha = { version = "0.9.0" } rand_core = "0.9.0" rayon = "1.10" regex = "1.10.4" diff --git a/src/uu/shuf/Cargo.toml b/src/uu/shuf/Cargo.toml index 135dc29f4..b271d9b9b 100644 --- a/src/uu/shuf/Cargo.toml +++ b/src/uu/shuf/Cargo.toml @@ -21,7 +21,9 @@ path = "src/shuf.rs" clap = { workspace = true } itoa = { workspace = true } rand = { workspace = true } +rand_chacha = { workspace = true } rand_core = { workspace = true } +sha3 = { workspace = true } uucore = { workspace = true } fluent = { workspace = true } diff --git a/src/uu/shuf/locales/en-US.ftl b/src/uu/shuf/locales/en-US.ftl index 477684fb2..de3221179 100644 --- a/src/uu/shuf/locales/en-US.ftl +++ b/src/uu/shuf/locales/en-US.ftl @@ -10,6 +10,7 @@ shuf-help-echo = treat each ARG as an input line shuf-help-input-range = treat each number LO through HI as an input line shuf-help-head-count = output at most COUNT lines shuf-help-output = write result to FILE instead of standard output +shuf-help-random-seed = seed with STRING for reproducible output shuf-help-random-source = get random bytes from FILE shuf-help-repeat = output lines can be repeated shuf-help-zero-terminated = line delimiter is NUL, not newline diff --git a/src/uu/shuf/src/compat_random_source.rs b/src/uu/shuf/src/compat_random_source.rs index 9d2d1e3b2..73a7191be 100644 --- a/src/uu/shuf/src/compat_random_source.rs +++ b/src/uu/shuf/src/compat_random_source.rs @@ -1,4 +1,9 @@ -use std::io::BufRead; +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use std::{io::BufRead, ops::RangeInclusive}; use uucore::error::{FromIo, UResult, USimpleError}; use uucore::translate; @@ -42,7 +47,7 @@ impl RandomSourceAdapter { } impl RandomSourceAdapter { - pub fn get_value(&mut self, at_most: u64) -> UResult { + fn generate_at_most(&mut self, at_most: u64) -> UResult { while self.entropy < at_most { let buf = self .reader @@ -88,10 +93,21 @@ impl RandomSourceAdapter { self.state %= num_possibilities; self.entropy %= num_possibilities; // I sure hope the compiler optimizes this tail call. - self.get_value(at_most) + self.generate_at_most(at_most) } } + pub fn choose_from_range(&mut self, range: RangeInclusive) -> UResult { + let offset = self.generate_at_most(*range.end() - *range.start())?; + Ok(*range.start() + offset) + } + + pub fn choose_from_slice(&mut self, vals: &[T]) -> UResult { + assert!(!vals.is_empty()); + let idx = self.generate_at_most(vals.len() as u64 - 1)? as usize; + Ok(vals[idx]) + } + pub fn shuffle<'a, T>(&mut self, vals: &'a mut [T], amount: usize) -> UResult<&'a mut [T]> { // Fisher-Yates shuffle. // TODO: GNU does something different if amount <= vals.len() and the input is stdin. @@ -99,7 +115,7 @@ impl RandomSourceAdapter { // No clue what they might do differently and why. let amount = amount.min(vals.len()); for idx in 0..amount { - let other_idx = self.get_value((vals.len() - idx - 1) as u64)? as usize + idx; + let other_idx = self.generate_at_most((vals.len() - idx - 1) as u64)? as usize + idx; vals.swap(idx, other_idx); } Ok(&mut vals[..amount]) diff --git a/src/uu/shuf/src/random_seed.rs b/src/uu/shuf/src/random_seed.rs new file mode 100644 index 000000000..f66ad62f8 --- /dev/null +++ b/src/uu/shuf/src/random_seed.rs @@ -0,0 +1,118 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use std::ops::RangeInclusive; + +use rand::{RngCore as _, SeedableRng as _}; +use rand_chacha::ChaCha12Rng; +use sha3::{Digest as _, Sha3_256}; + +/// Reproducible seeded random number generation. +/// +/// The behavior should stay the same between releases, so don't change it without +/// a very good reason. +/// +/// # How it works +/// +/// - Take a Unicode string as the seed. +/// +/// - Encode this seed as UTF-8. +/// +/// - Take the SHA3-256 hash of the encoded seed. +/// +/// - Use that hash as the input for a [`rand_chacha`] ChaCha12 RNG. +/// (We don't touch the nonce, so that's probably zero.) +/// +/// - Take 64-bit samples from the RNG. +/// +/// - Use Lemire's method to generate uniformly distributed integers and: +/// +/// - With --repeat, use these to pick elements from ranges. +/// +/// - Without --repeat, use these to do left-to-right modern Fisher-Yates. +/// +/// - Or for --input-range without --repeat, do whatever NonrepeatingIterator does. +/// (We may want to change that. Watch this space.) +/// +/// # Why it works like this +/// +/// - Unicode string: Greatest common denominator between platforms. Windows doesn't +/// let you pass raw bytes as a CLI argument and that would be bad practice anyway. +/// A decimal or hex number would work but this is much more flexible without being +/// unmanageable. +/// +/// (Footgun: if the user passes a filename we won't read from the file but the +/// command will run anyway.) +/// +/// - UTF-8: That's what Rust likes and it's the least unreasonable Unicode encoding. +/// +/// - SHA3-256: We want to make good use of the entire user input and SHA-3 is +/// state of the art. ChaCha12 takes a 256-bit seed. +/// +/// - ChaCha12: [`rand`]'s default rng as of writing. Seems state of the art. +/// +/// - 64-bit samples: We could often get away with 32-bit samples but let's keep things +/// simple and only use one width. (There doesn't seem to be much of a performance hit.) +/// +/// - Lemire, Fisher-Yates: These are very easy to implement and maintain ourselves. +/// `rand` provides fancier implementations but only promises reproducibility within +/// patch releases: +/// +/// Strictly speaking even `ChaCha12` is subject to breakage. But since it's a very +/// specific algorithm I assume it's safe in practice. +pub struct SeededRng(Box); + +impl SeededRng { + pub fn new(seed: &str) -> Self { + let mut hasher = Sha3_256::new(); + hasher.update(seed.as_bytes()); + let seed = hasher.finalize(); + let seed = seed.as_slice().try_into().unwrap(); + Self(Box::new(rand_chacha::ChaCha12Rng::from_seed(seed))) + } + + #[allow(clippy::many_single_char_names)] // use original lemire names for easy comparison + fn generate_at_most(&mut self, at_most: u64) -> u64 { + if at_most == u64::MAX { + return self.0.next_u64(); + } + + // https://lemire.me/blog/2019/06/06/nearly-divisionless-random-integer-generation-on-various-systems/ + let s: u64 = at_most + 1; + let mut x: u64 = self.0.next_u64(); + let mut m: u128 = u128::from(x) * u128::from(s); + let mut l: u64 = m as u64; + if l < s { + let t: u64 = s.wrapping_neg() % s; + while l < t { + x = self.0.next_u64(); + m = u128::from(x) * u128::from(s); + l = m as u64; + } + } + (m >> 64) as u64 + } + + pub fn choose_from_range(&mut self, range: RangeInclusive) -> u64 { + let offset = self.generate_at_most(*range.end() - *range.start()); + *range.start() + offset + } + + pub fn choose_from_slice(&mut self, vals: &[T]) -> T { + assert!(!vals.is_empty()); + let idx = self.generate_at_most(vals.len() as u64 - 1) as usize; + vals[idx] + } + + pub fn shuffle<'a, T>(&mut self, vals: &'a mut [T], amount: usize) -> &'a mut [T] { + // Fisher-Yates shuffle. + let amount = amount.min(vals.len()); + for idx in 0..amount { + let other_idx = self.generate_at_most((vals.len() - idx - 1) as u64) as usize + idx; + vals.swap(idx, other_idx); + } + &mut vals[..amount] + } +} diff --git a/src/uu/shuf/src/shuf.rs b/src/uu/shuf/src/shuf.rs index 9a3e3afd4..e2cb2e958 100644 --- a/src/uu/shuf/src/shuf.rs +++ b/src/uu/shuf/src/shuf.rs @@ -5,16 +5,20 @@ // spell-checker:ignore (ToDO) cmdline evec nonrepeating seps shufable rvec fdata -use clap::builder::ValueParser; -use clap::{Arg, ArgAction, Command}; -use rand::Rng; -use rand::seq::{IndexedRandom, SliceRandom}; use std::ffi::{OsStr, OsString}; use std::fs::File; use std::io::{BufReader, BufWriter, Error, Read, Write, stdin, stdout}; use std::ops::RangeInclusive; use std::path::{Path, PathBuf}; use std::str::FromStr; + +use clap::{Arg, ArgAction, Command, builder::ValueParser}; +use rand::rngs::ThreadRng; +use rand::{ + Rng, + seq::{IndexedRandom, SliceRandom}, +}; + use uucore::display::{OsWrite, Quotable}; use uucore::error::{FromIo, UResult, USimpleError, UUsageError}; use uucore::format_usage; @@ -22,8 +26,11 @@ use uucore::translate; mod compat_random_source; mod nonrepeating_iterator; +mod random_seed; +use compat_random_source::RandomSourceAdapter; use nonrepeating_iterator::NonrepeatingIterator; +use random_seed::SeededRng; enum Mode { Default(PathBuf), @@ -36,17 +43,24 @@ const BUF_SIZE: usize = 64 * 1024; struct Options { head_count: u64, output: Option, - random_source: Option, + random_source: RandomSource, repeat: bool, sep: u8, } +enum RandomSource { + None, + Seed(String), + File(PathBuf), +} + mod options { pub static ECHO: &str = "echo"; pub static INPUT_RANGE: &str = "input-range"; pub static HEAD_COUNT: &str = "head-count"; pub static OUTPUT: &str = "output"; pub static RANDOM_SOURCE: &str = "random-source"; + pub static RANDOM_SEED: &str = "random-seed"; pub static REPEAT: &str = "repeat"; pub static ZERO_TERMINATED: &str = "zero-terminated"; pub static FILE_OR_ARGS: &str = "file-or-args"; @@ -80,6 +94,14 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { Mode::Default(file.into()) }; + let random_source = if let Some(filename) = matches.get_one(options::RANDOM_SOURCE).cloned() { + RandomSource::File(filename) + } else if let Some(seed) = matches.get_one(options::RANDOM_SEED).cloned() { + RandomSource::Seed(seed) + } else { + RandomSource::None + }; + let options = Options { // GNU shuf takes the lowest value passed, so we imitate that. // It's probably a bug or an implementation artifact though. @@ -92,7 +114,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { .min() .unwrap_or(u64::MAX), output: matches.get_one(options::OUTPUT).cloned(), - random_source: matches.get_one(options::RANDOM_SOURCE).cloned(), + random_source, repeat: matches.get_flag(options::REPEAT), sep: if matches.get_flag(options::ZERO_TERMINATED) { b'\0' @@ -120,14 +142,15 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } let mut rng = match options.random_source { - Some(ref r) => { + RandomSource::None => WrappedRng::Default(rand::rng()), + RandomSource::Seed(ref seed) => WrappedRng::Seed(SeededRng::new(seed)), + RandomSource::File(ref r) => { let file = File::open(r).map_err_context( || translate!("shuf-error-failed-to-open-random-source", "file" => r.quote()), )?; let file = BufReader::new(file); - WrappedRng::RngFile(compat_random_source::RandomSourceAdapter::new(file)) + WrappedRng::File(compat_random_source::RandomSourceAdapter::new(file)) } - None => WrappedRng::RngDefault(rand::rng()), }; match mode { @@ -191,6 +214,15 @@ pub fn uu_app() -> Command { .value_parser(ValueParser::path_buf()) .value_hint(clap::ValueHint::FilePath), ) + .arg( + Arg::new(options::RANDOM_SEED) + .long(options::RANDOM_SEED) + .value_name("STRING") + .help(translate!("shuf-help-random-seed")) + .value_parser(ValueParser::string()) + .value_hint(clap::ValueHint::Other) + .conflicts_with(options::RANDOM_SOURCE), + ) .arg( Arg::new(options::RANDOM_SOURCE) .long(options::RANDOM_SOURCE) @@ -402,36 +434,33 @@ fn parse_range(input_range: &str) -> Result, String> { } enum WrappedRng { - RngDefault(rand::rngs::ThreadRng), - RngFile(compat_random_source::RandomSourceAdapter>), + Default(ThreadRng), + Seed(SeededRng), + File(RandomSourceAdapter>), } impl WrappedRng { fn choose(&mut self, vals: &[T]) -> UResult { match self { - Self::RngDefault(rng) => Ok(*vals.choose(rng).unwrap()), - Self::RngFile(adapter) => { - assert!(!vals.is_empty()); - let idx = adapter.get_value(vals.len() as u64 - 1)? as usize; - Ok(vals[idx]) - } + Self::Default(rng) => Ok(*vals.choose(rng).unwrap()), + Self::Seed(rng) => Ok(rng.choose_from_slice(vals)), + Self::File(rng) => rng.choose_from_slice(vals), } } fn shuffle<'a, T>(&mut self, vals: &'a mut [T], amount: usize) -> UResult<&'a mut [T]> { match self { - Self::RngDefault(rng) => Ok(vals.partial_shuffle(rng, amount).0), - Self::RngFile(adapter) => adapter.shuffle(vals, amount), + Self::Default(rng) => Ok(vals.partial_shuffle(rng, amount).0), + Self::Seed(rng) => Ok(rng.shuffle(vals, amount)), + Self::File(rng) => rng.shuffle(vals, amount), } } fn choose_from_range(&mut self, range: RangeInclusive) -> UResult { match self { - Self::RngDefault(rng) => Ok(rng.random_range(range)), - Self::RngFile(adapter) => { - let offset = adapter.get_value(*range.end() - *range.start())?; - Ok(*range.start() + offset) - } + Self::Default(rng) => Ok(rng.random_range(range)), + Self::Seed(rng) => Ok(rng.choose_from_range(range)), + Self::File(rng) => rng.choose_from_range(range), } } } diff --git a/tests/by-util/test_shuf.rs b/tests/by-util/test_shuf.rs index 1b5d2a99c..b419419ec 100644 --- a/tests/by-util/test_shuf.rs +++ b/tests/by-util/test_shuf.rs @@ -1018,3 +1018,52 @@ fn test_gnu_compat_range_no_repeat() { .no_stderr() .stdout_is("10\n2\n8\n7\n3\n9\n6\n5\n1\n4\n"); } + +// Test reproducibility of --random-seed. +// These results are arbitrary but they should not change unless we choose to break compatibility. + +#[test] +fn test_seed_args_repeat() { + new_ucmd!() + .arg("--random-seed=🌱") + .arg("-e") + .arg("-r") + .arg("-n10") + .args(&["foo", "bar", "baz", "qux"]) + .succeeds() + .no_stderr() + .stdout_is("qux\nbar\nbaz\nfoo\nbaz\nqux\nqux\nfoo\nqux\nqux\n"); +} + +#[test] +fn test_seed_args_no_repeat() { + new_ucmd!() + .arg("--random-seed=🌱") + .arg("-e") + .args(&["foo", "bar", "baz", "qux"]) + .succeeds() + .no_stderr() + .stdout_is("qux\nbaz\nfoo\nbar\n"); +} + +#[test] +fn test_seed_range_repeat() { + new_ucmd!() + .arg("--random-seed=🦀") + .arg("-r") + .arg("-i1-99") + .arg("-n10") + .succeeds() + .no_stderr() + .stdout_is("60\n44\n38\n41\n63\n43\n31\n71\n46\n90\n"); +} + +#[test] +fn test_seed_range_no_repeat() { + new_ucmd!() + .arg("--random-seed=12345") + .arg("-i1-10") + .succeeds() + .no_stderr() + .stdout_is("8\n9\n5\n10\n1\n2\n4\n7\n3\n6\n"); +}