From f7365fa4226a99e6c5bae51c378ffe633723c17b Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sat, 24 May 2025 12:46:32 +0300 Subject: [PATCH 01/17] Add benchmark and comparison scripts --- tests/fixtures/sed/script/http-log-redact.sed | 4 + util/bench-compare.sh | 26 ++++ util/benchmark.sh | 137 ++++++++++++++++++ 3 files changed, 167 insertions(+) create mode 100644 tests/fixtures/sed/script/http-log-redact.sed create mode 100755 util/bench-compare.sh create mode 100755 util/benchmark.sh diff --git a/tests/fixtures/sed/script/http-log-redact.sed b/tests/fixtures/sed/script/http-log-redact.sed new file mode 100644 index 0000000..600ae9f --- /dev/null +++ b/tests/fixtures/sed/script/http-log-redact.sed @@ -0,0 +1,4 @@ +#!/ +s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[REDACTED_IP]/; +s/\[[0-9]{2}\/[A-Za-z]{3}\/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} [+-][0-9]{4}\]/[TIMESTAMP]/; +s/"Mozilla[^"]*"/"[UA]"/ diff --git a/util/bench-compare.sh b/util/bench-compare.sh new file mode 100755 index 0000000..da0fe27 --- /dev/null +++ b/util/bench-compare.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# +# Compare the results of two benchmarks reporting fastest for each task +# + + +set -eu + +if [ $# -ne 2 ] ; then + echo "Usage: $(basename $0) results-1 results-2" 1>&2 + + exit 1 +fi + +paste -d , "$1" "$2" | + sed "s/^/$1,$2,/" | + + awk -F, 'FNR != 1 { + printf("%22s ", $3); + if ($4 / $12 > 1.1) + printf("%12s is %5.1f times faster than %s\n", $2, $4 / $12, $1); + else if ($12 / $4 > 1.1) + printf("%12s is %5.1f times faster than %s\n", $1, $12 / $4, $2); + else + printf("%12s is similarly fast as %s\n", $1, $2); + }' diff --git a/util/benchmark.sh b/util/benchmark.sh new file mode 100755 index 0000000..ddade41 --- /dev/null +++ b/util/benchmark.sh @@ -0,0 +1,137 @@ +#!/bin/sh + +set -eu + +if [ $# -ne 2 ] ; then + cat <&2 +Usage: $(basename $0) sed-command output-file" + +Examples: +$(basename $0) 'target/release/sedapp sed' rust-sed +$(basename $0) /usr/bin/sed gnu-sed +EOF + exit 1 +fi + +PROG="$1" +OUT="$2" + +SCRIPTS=tests/fixtures/sed/script + + +# Run hyperfine with the specified name and command and collect the results. +bench_run() +{ + hyperfine --command-name "$1" --warmup 2 --export-csv out.csv "$2" + + # Append the results sans-heading + sed 1d out.csv >>"$OUT" + + rm out.csv +} + +echo 'command,mean,stddev,median,user,system,min,max' >"$OUT" + +# Short line processing +awk 'BEGIN { for (i = 0; i < 50000000; i++) { print i } }' > lines.txt + +# No operation +bench_run no-op-short "$PROG '' lines.txt" + +# Log file processing + +# Create an access.log file with the specified number of lines +create_access_log() +{ + awk 'BEGIN { + for (i = 0; i < 5000000; i++) { + printf("192.168.%d.%d - - [01/Jan/2024:00:00:00 +0000] \"GET /index.html HTTP/1.1\" 200 1234 \"-\" \"Mozilla/5.0\"\n", int(i/256)%256, i%256); + } + }' > access.log +} + +# Create long file for simple commands +create_access_log 5000000 + +# No operation +bench_run access-log-no-op "$PROG '' access.log" + +# No substitution +bench_run access-log-no-subst "$PROG s/Chrome/Chromium/ access.log" + +# Substitution +bench_run access-log-subst "$PROG s/Mozilla/Chromium/ access.log" + +# No deletion +bench_run access-log-no-del "$PROG /Chrome/d access.log" + +# Full deletion +bench_run access-log-all-del "$PROG /Mozilla/d access.log" + +# Create shorter file for more complex commands +create_access_log 50000 + +# Transliteration +bench_run access-log-translit "$PROG y/0123456789/9876543210/ access.log" + +# Multiple substitutions +bench_run access-log-complex-sub "$PROG -f $SCRIPTS/http-log-redact.sed access.log" + +rm access.log + +# Remove \r +awk 'BEGIN { + for (i = 0; i < 1500000; i++) { + printf("line %d with windows endings\r\n", i); + } +}' > legacy_input.txt + +bench_run remove-cr "$PROG 's/\r$//' legacy_input.txt" + +rm legacy_input.txt + +# Genomic data cleanup + +awk 'BEGIN { + for (i = 0; i < 1000000; i++) { + chr = "chr" (1 + i % 22); + printf("%s\t%d\t%s\t.\t%s\t.\t.\n", chr, i * 100, (i % 2 ? "A" : "T"), (i % 2 ? "G" : "C")); + } +}' > genome.tsv + +CMD='/^#/d; s/\t\./\tNA/g; s/\.$/NA/' +bench_run genome-subst "$PROG '$CMD' genome.tsv" + +rm -f genome.tsv + +# Number fixups: remove thousands separator, change , into . +awk 'BEGIN { + for (i = 0; i < 700000; i++) { + euros = int(i % 10000); + cents = int(i % 100); + thousands = int(euros / 1000); + remainder = euros % 1000; + printf("%d.%03d,%02d\n", thousands, remainder, cents); + } +}' > finance.csv + +CMD='s/\([0-9]\)\.\([0-9]\)/\1\2/g;s/\([0-9]\),\([0-9]\)/\1.\2/g' +bench_run number-fix "$PROG '$CMD' finance.csv" + +rm -f finance.csv + +# Long script compilation +for i in $(seq 1 99) ; do + awk -v tag="$i" '{ gsub(/[tb:] [[:alnum:]_]+/, "&" tag); print }' $SCRIPTS/math.sed +done >long_script.sed + +bench_run long-script "echo -n '' | $PROG -f long_script.sed " + +rm long_script.sed + + +# Towers of Hanoi +bench_run hanoi "echo ':abcdefghijkl: : :' | $PROG -f $SCRIPTS/hanoi.sed" + +# Arbitrary precision arithmetic +bench_run factorial "echo 30\! | $PROG -f $SCRIPTS/math.sed" From b31d9d15dbcf5d26cd29726dd6fd66fe28f9ca26 Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sat, 24 May 2025 12:56:18 +0300 Subject: [PATCH 02/17] Use memchr to speed up line splitting This improves performance up to 1.5 times, with no downside. no-op-short rust-sed is similarly fast as rust-sed-f0 access-log-no-op rust-sed is 1.5 times faster than rust-sed-f0 access-log-no-subst rust-sed is similarly fast as rust-sed-f0 access-log-subst rust-sed is similarly fast as rust-sed-f0 access-log-no-del rust-sed is 1.1 times faster than rust-sed-f0 access-log-all-del rust-sed is similarly fast as rust-sed-f0 access-log-translit rust-sed is 1.2 times faster than rust-sed-f0 access-log-complex-sub rust-sed is similarly fast as rust-sed-f0 remove-cr rust-sed is similarly fast as rust-sed-f0 genome-subst rust-sed is similarly fast as rust-sed-f0 number-fix rust-sed is 1.1 times faster than rust-sed-f0 long-script rust-sed is similarly fast as rust-sed-f0 hanoi rust-sed is similarly fast as rust-sed-f0 factorial rust-sed is similarly fast as rust-sed-f0 --- Cargo.lock | 2 ++ Cargo.toml | 2 ++ src/uu/sed/Cargo.toml | 1 + src/uu/sed/src/fast_io.rs | 11 +++++++---- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e87e1f3..15c8473 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -703,6 +703,7 @@ dependencies = [ "ctor", "fancy-regex", "libc", + "memchr", "memmap2", "phf", "phf_codegen", @@ -896,6 +897,7 @@ version = "0.0.1" dependencies = [ "clap", "fancy-regex", + "memchr", "memmap2", "regex", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 871f72c..61a7d8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ clap_complete = "4.5" clap_mangen = "0.2" fancy-regex = "0.14.0" libc = "0.2.153" +memchr = "2.7.4" memmap2 = "0.9" phf = "0.11.2" phf_codegen = "0.11.2" @@ -56,6 +57,7 @@ clap_complete = { workspace = true } clap_mangen = { workspace = true } ctor = "0.4.1" fancy-regex = { workspace = true } +memchr = { workspace = true } memmap2.workspace = true phf = { workspace = true } sed = { optional = true, version = "0.0.1", package = "uu_sed", path = "src/uu/sed" } diff --git a/src/uu/sed/Cargo.toml b/src/uu/sed/Cargo.toml index 6d94616..8487ab2 100644 --- a/src/uu/sed/Cargo.toml +++ b/src/uu/sed/Cargo.toml @@ -15,6 +15,7 @@ categories = ["command-line-utilities"] [dependencies] clap = { workspace = true } fancy-regex = { workspace = true } +memchr = { workspace = true } regex = { workspace = true } tempfile = { workspace = true } memmap2 = { workspace = true } diff --git a/src/uu/sed/src/fast_io.rs b/src/uu/sed/src/fast_io.rs index 36bb53e..1cc7e12 100644 --- a/src/uu/sed/src/fast_io.rs +++ b/src/uu/sed/src/fast_io.rs @@ -14,6 +14,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +use memchr::memchr; #[cfg(unix)] use memmap2::Mmap; @@ -69,10 +70,12 @@ impl<'a> MmapLineCursor<'a> { } let start = self.pos; - let mut end = start; - while end < self.data.len() && self.data[end] != b'\n' { - end += 1; - } + + let mut end = if let Some(pos) = memchr(b'\n', &self.data[start..]) { + pos + start + } else { + self.data.len() + }; if end < self.data.len() { end += 1; // include \n in full span From a4cee5127d91eeb5aab24bb23c04cdb451639982 Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sat, 24 May 2025 16:15:24 +0300 Subject: [PATCH 03/17] Handle failed executions --- util/bench-compare.sh | 6 +++++- util/benchmark.sh | 12 ++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/util/bench-compare.sh b/util/bench-compare.sh index da0fe27..0ee61b8 100755 --- a/util/bench-compare.sh +++ b/util/bench-compare.sh @@ -17,7 +17,11 @@ paste -d , "$1" "$2" | awk -F, 'FNR != 1 { printf("%22s ", $3); - if ($4 / $12 > 1.1) + if (!$4) + printf("%12s encountered an error\n", $1); + else if (!$12) + printf("%12s encountered an error\n", $2); + else if ($4 / $12 > 1.1) printf("%12s is %5.1f times faster than %s\n", $2, $4 / $12, $1); else if ($12 / $4 > 1.1) printf("%12s is %5.1f times faster than %s\n", $1, $12 / $4, $2); diff --git a/util/benchmark.sh b/util/benchmark.sh index ddade41..ebbb02a 100755 --- a/util/benchmark.sh +++ b/util/benchmark.sh @@ -22,14 +22,18 @@ SCRIPTS=tests/fixtures/sed/script # Run hyperfine with the specified name and command and collect the results. bench_run() { - hyperfine --command-name "$1" --warmup 2 --export-csv out.csv "$2" - - # Append the results sans-heading - sed 1d out.csv >>"$OUT" + if hyperfine --command-name "$1" --warmup 2 --export-csv out.csv "$2" ; then + # Output the results sans-heading. + sed 1d out.csv >>"$OUT" + else + # Unable to run; output a named empty record. + echo "$1,,,,,,," >>"$OUT" + fi rm out.csv } +# Shared heading echo 'command,mean,stddev,median,user,system,min,max' >"$OUT" # Short line processing From 25b84bda239d04e0b2261415df2740e06257876f Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sat, 24 May 2025 18:38:26 +0300 Subject: [PATCH 04/17] Check at runtime for invalid s/// group regerences --- src/uu/sed/src/command.rs | 138 +++++++++++++++++++++---------------- src/uu/sed/src/compiler.rs | 2 +- 2 files changed, 80 insertions(+), 60 deletions(-) diff --git a/src/uu/sed/src/command.rs b/src/uu/sed/src/command.rs index 24a8038..d5643ea 100644 --- a/src/uu/sed/src/command.rs +++ b/src/uu/sed/src/command.rs @@ -19,7 +19,7 @@ use std::cell::RefCell; use std::collections::HashMap; use std::path::PathBuf; // For file descriptors and equivalent use std::rc::Rc; -use uucore::error::UResult; +use uucore::error::{UResult, USimpleError}; #[derive(Debug, Default, Clone)] /// Compilation and processing options provided mostly through the @@ -124,16 +124,34 @@ pub enum ReplacementPart { /// All specified replacements for an RE pub struct ReplacementTemplate { pub parts: Vec, + max_group_number: usize, // Maximum referenced group number (e.g. 8 for \8) } impl Default for ReplacementTemplate { /// Create an empty template. fn default() -> Self { - ReplacementTemplate { parts: Vec::new() } + ReplacementTemplate::new(Vec::new()) } } impl ReplacementTemplate { + /// Construct from the parts + pub fn new(parts: Vec) -> Self { + let max_group_number = parts + .iter() + .filter_map(|part| match part { + ReplacementPart::Group(n) => Some(*n), + _ => None, + }) + .max() + .unwrap_or(0); + + Self { + parts, + max_group_number: max_group_number.try_into().unwrap(), + } + } + /// Apply the template to the given RE captures. /// Example: /// let result = regex.replace_all(input, |caps: &Captures| { @@ -142,6 +160,18 @@ impl ReplacementTemplate { pub fn apply(&self, caps: &Captures) -> UResult { let mut result = String::new(); + // Invalid group numbers may end here through (unkown at compile time) + // reused REs. + if self.max_group_number > caps.len() - 1 { + return Err(USimpleError::new( + 2, + format!( + "invalid reference \\{} on `s' command's RHS", + self.max_group_number + ), + )); + } + for part in &self.parts { match part { ReplacementPart::Literal(s) => result.push_str(s), @@ -151,7 +181,6 @@ impl ReplacementTemplate { } ReplacementPart::Group(n) => { - // Compilation guarantees we only get valid group numbers result.push_str( caps.get((*n).try_into().unwrap()) .map_or("", |m| m.as_str()), @@ -162,21 +191,6 @@ impl ReplacementTemplate { Ok(result) } - - /// Returns the highest capture group number referenced in this template. - pub fn max_group_number(&self) -> u32 { - self.parts - .iter() - .filter_map(|part| { - if let ReplacementPart::Group(n) = part { - Some(*n) - } else { - None - } - }) - .max() - .unwrap_or(0) - } } #[derive(Debug, Default)] @@ -345,9 +359,7 @@ mod tests { #[test] // s/abc/hello/ fn test_literal_only() { - let template = ReplacementTemplate { - parts: vec![ReplacementPart::Literal("hello".into())], - }; + let template = ReplacementTemplate::new(vec![ReplacementPart::Literal("hello".into())]); let caps = caps_for("abc", "abc"); let result = template.apply(&caps).unwrap(); @@ -357,12 +369,10 @@ mod tests { #[test] // s/foo\d+/got: &/ fn test_whole_match() { - let template = ReplacementTemplate { - parts: vec![ - ReplacementPart::Literal("got: ".into()), - ReplacementPart::WholeMatch, - ], - }; + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("got: ".into()), + ReplacementPart::WholeMatch, + ]); let caps = caps_for(r"foo\d+", "foo42"); let result = template.apply(&caps).unwrap(); @@ -372,12 +382,10 @@ mod tests { #[test] // s/foo(\d+)/number: \1/ fn test_backreference() { - let template = ReplacementTemplate { - parts: vec![ - ReplacementPart::Literal("number: ".into()), - ReplacementPart::Group(1), - ], - }; + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("number: ".into()), + ReplacementPart::Group(1), + ]); let caps = caps_for(r"foo(\d+)", "foo42"); let result = template.apply(&caps).unwrap(); @@ -387,45 +395,57 @@ mod tests { #[test] // s/(\w+):(\d+)/key: \1, value: \2/ fn test_multiple_parts() { - let template = ReplacementTemplate { - parts: vec![ - ReplacementPart::Literal("key: ".into()), - ReplacementPart::Group(1), - ReplacementPart::Literal(", value: ".into()), - ReplacementPart::Group(2), - ], - }; + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("key: ".into()), + ReplacementPart::Group(1), + ReplacementPart::Literal(", value: ".into()), + ReplacementPart::Group(2), + ]); let caps = caps_for(r"(\w+):(\d+)", "x:123"); let result = template.apply(&caps).unwrap(); assert_eq!(result, "key: x, value: 123"); } + #[test] + // s/(\w+):(\d+)/key: \1, value: \3/ + fn test_invalid_group() { + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("key: ".into()), + ReplacementPart::Group(1), + ReplacementPart::Literal(", value: ".into()), + ReplacementPart::Group(3), + ]); + let caps = caps_for(r"(\w+):(\d+)", "x:123"); + + let result = template.apply(&caps); + assert!(result.is_err()); + + let msg = result.unwrap_err().to_string(); + assert!(msg.contains("invalid reference \\3")); + } + // max_group_number #[test] fn test_max_group_number_with_groups() { - let template = ReplacementTemplate { - parts: vec![ - ReplacementPart::Literal("a".into()), - ReplacementPart::Group(2), - ReplacementPart::WholeMatch, - ReplacementPart::Group(5), - ReplacementPart::Literal("z".into()), - ], - }; - assert_eq!(template.max_group_number(), 5); + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("a".into()), + ReplacementPart::Group(2), + ReplacementPart::WholeMatch, + ReplacementPart::Group(5), + ReplacementPart::Literal("z".into()), + ]); + assert_eq!(template.max_group_number, 5); } #[test] fn test_max_group_number_without_groups() { - let template = ReplacementTemplate { - parts: vec![ - ReplacementPart::Literal("no".into()), - ReplacementPart::WholeMatch, - ReplacementPart::Literal("groups".into()), - ], - }; - assert_eq!(template.max_group_number(), 0); + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("no".into()), + ReplacementPart::WholeMatch, + ReplacementPart::Literal("groups".into()), + ]); + assert_eq!(template.max_group_number, 0); } // Transliteration diff --git a/src/uu/sed/src/compiler.rs b/src/uu/sed/src/compiler.rs index cdfc135..0dd7003 100644 --- a/src/uu/sed/src/compiler.rs +++ b/src/uu/sed/src/compiler.rs @@ -762,7 +762,7 @@ pub fn compile_replacement( if !literal.is_empty() { parts.push(ReplacementPart::Literal(literal)); } - return Ok(ReplacementTemplate { parts }); + return Ok(ReplacementTemplate::new(parts)); } c => { From cbe314d01b2731ad54a42c0bddf4495a8bceb03f Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sat, 24 May 2025 19:00:54 +0300 Subject: [PATCH 05/17] Catch invalid group references at compile time --- src/uu/sed/src/command.rs | 2 +- src/uu/sed/src/compiler.rs | 29 ++++++++++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/uu/sed/src/command.rs b/src/uu/sed/src/command.rs index d5643ea..6e06f30 100644 --- a/src/uu/sed/src/command.rs +++ b/src/uu/sed/src/command.rs @@ -124,7 +124,7 @@ pub enum ReplacementPart { /// All specified replacements for an RE pub struct ReplacementTemplate { pub parts: Vec, - max_group_number: usize, // Maximum referenced group number (e.g. 8 for \8) + pub max_group_number: usize, // Highest used group number (e.g. 8 for \8) } impl Default for ReplacementTemplate { diff --git a/src/uu/sed/src/compiler.rs b/src/uu/sed/src/compiler.rs index 0dd7003..ad21e76 100644 --- a/src/uu/sed/src/compiler.rs +++ b/src/uu/sed/src/compiler.rs @@ -816,11 +816,21 @@ fn compile_subst_command( ); } - if pattern.is_empty() { - subst.regex = None; // Will be reuse saved RE at runtime. - } else { - // Compile regex with now known ignore_case flag. - subst.regex = compile_regex(lines, line, &pattern, context, subst.ignore_case)?; + // Compile regex with now known ignore_case flag. + subst.regex = compile_regex(lines, line, &pattern, context, subst.ignore_case)?; + + // Catch invalid group references at compile time, if possible. + if let Some(regex) = &subst.regex { + if subst.replacement.max_group_number > regex.captures_len() - 1 { + return compilation_error( + lines, + line, + format!( + "invalid reference \\{} on `s' command's RHS", + subst.replacement.max_group_number + ), + ); + } } cmd.data = CommandData::Substitution(subst); @@ -1989,6 +1999,15 @@ mod tests { } } + #[test] + fn test_compile_subst_invalid_group_reference() { + let (mut lines, mut chars) = make_providers(r"s/f(o)o/\2/"); + let mut cmd = Command::default(); + + let err = compile_subst_command(&mut lines, &mut chars, &mut cmd, &ctx()).unwrap_err(); + assert!(err.to_string().contains("invalid reference \\2")); + } + // bre_to_ere #[test] fn test_bre_group_translation() { From eb64104a21093f18c987f0c97fd73272bf4deed1 Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sat, 24 May 2025 19:12:49 +0300 Subject: [PATCH 06/17] Support replacement group \0 as synonym for & --- README.md | 1 + src/uu/sed/src/compiler.rs | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 365064d..8cbd8c9 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ cargo run --release * In addition to `\n`, other escape sequences (octal, hex, C) are supported in the strings of the `y` command. Under POSIX these yield undefined behavior. +* The substitution command replacement group `\0` is a synonym for &. ### Supported BSD and GNU extensions * The second address in a range can be specified as a relative address with +N. diff --git a/src/uu/sed/src/compiler.rs b/src/uu/sed/src/compiler.rs index ad21e76..5fdcfc3 100644 --- a/src/uu/sed/src/compiler.rs +++ b/src/uu/sed/src/compiler.rs @@ -706,14 +706,18 @@ pub fn compile_replacement( } match line.current() { - // \1 - \9 - c @ '1'..='9' => { + // \0 - \9 + c @ '0'..='9' => { let ref_num = c.to_digit(10).unwrap(); if !literal.is_empty() { parts.push(ReplacementPart::Literal(std::mem::take(&mut literal))); } - parts.push(ReplacementPart::Group(ref_num)); + if ref_num == 0 { + parts.push(ReplacementPart::WholeMatch); + } else { + parts.push(ReplacementPart::Group(ref_num)); + } line.advance(); } @@ -1806,6 +1810,18 @@ mod tests { assert!(matches!(&template.parts[1], ReplacementPart::WholeMatch)); } + #[test] + fn test_compile_replacement_whole_match_synonym() { + let (mut lines, mut chars) = make_providers(r"/The match was: \0/"); + let template = compile_replacement(&mut lines, &mut chars).unwrap(); + + assert_eq!(template.parts.len(), 2); + assert!( + matches!(&template.parts[0], ReplacementPart::Literal(s) if s == "The match was: ") + ); + assert!(matches!(&template.parts[1], ReplacementPart::WholeMatch)); + } + #[test] fn test_compile_replacement_ampersand() { let (mut lines, mut chars) = make_providers("/Simon \\& Garfunkel/"); From 7868f4f4572fead68756ae8d064317220251aa6a Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sat, 24 May 2025 20:26:34 +0300 Subject: [PATCH 07/17] Early exit in substitution This improves performance by 10% in the following benchmark case. Performance in all others remains the same. access-log-subst rust-sed is 1.1 times faster than rust-sed-eb64104 --- src/uu/sed/src/processor.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/uu/sed/src/processor.rs b/src/uu/sed/src/processor.rs index 7ac819b..bd2522a 100644 --- a/src/uu/sed/src/processor.rs +++ b/src/uu/sed/src/processor.rs @@ -225,6 +225,11 @@ fn substitute( } last_end = m.end(); + + // Early exit if only a specific occurrence (likely 1) needed replacing. + if count == sub.occurrence { + break; + } } // Handle substitution success. From f18f772f31b21660df54474dc3f0872ce53c8612 Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sat, 24 May 2025 22:51:35 +0300 Subject: [PATCH 08/17] Try to specialize regex retrieval In theory, find, find_iter, captures, and captures_iter according to the values of sub.occurrence, sub.replacement.max_group_number could allow for better performance. In practice, specializing for the easiest case to use find did not show any improvement. no-op-short rust-sed is similarly fast as rust-sed-7868f4f access-log-no-op rust-sed is similarly fast as rust-sed-7868f4f access-log-no-subst rust-sed is similarly fast as rust-sed-7868f4f access-log-subst rust-sed is similarly fast as rust-sed-7868f4f access-log-no-del rust-sed is similarly fast as rust-sed-7868f4f access-log-all-del rust-sed is similarly fast as rust-sed-7868f4f access-log-translit rust-sed is similarly fast as rust-sed-7868f4f access-log-complex-sub rust-sed is similarly fast as rust-sed-7868f4f remove-cr rust-sed is similarly fast as rust-sed-7868f4f genome-subst rust-sed is similarly fast as rust-sed-7868f4f number-fix rust-sed is similarly fast as rust-sed-7868f4f long-script rust-sed is similarly fast as rust-sed-7868f4f hanoi rust-sed is similarly fast as rust-sed-7868f4f factorial rust-sed is similarly fast as rust-sed-7868f4f Consequently, this commit just documents the change and its results, and will be reverted. --- src/uu/sed/src/command.rs | 36 ++++++++++++++----- src/uu/sed/src/processor.rs | 69 ++++++++++++++++++++++++------------- 2 files changed, 72 insertions(+), 33 deletions(-) diff --git a/src/uu/sed/src/command.rs b/src/uu/sed/src/command.rs index 6e06f30..e2fb162 100644 --- a/src/uu/sed/src/command.rs +++ b/src/uu/sed/src/command.rs @@ -13,7 +13,7 @@ use crate::named_writer::NamedWriter; -use fancy_regex::{Captures, Regex}; +use fancy_regex::{Captures, Match, Regex}; use std::borrow::Cow; use std::cell::RefCell; use std::collections::HashMap; @@ -155,9 +155,9 @@ impl ReplacementTemplate { /// Apply the template to the given RE captures. /// Example: /// let result = regex.replace_all(input, |caps: &Captures| { - /// template.apply(caps) }); + /// template.apply_captures(caps) }); /// Returns an error if a backreference in the template was not matched by the RE. - pub fn apply(&self, caps: &Captures) -> UResult { + pub fn apply_captures(&self, caps: &Captures) -> UResult { let mut result = String::new(); // Invalid group numbers may end here through (unkown at compile time) @@ -191,6 +191,24 @@ impl ReplacementTemplate { Ok(result) } + + /// Apply the template to the given RE single match. + pub fn apply_match(&self, m: &Match) -> String { + let mut result = String::new(); + + for part in &self.parts { + match part { + ReplacementPart::Literal(s) => result.push_str(s), + + ReplacementPart::WholeMatch => result.push_str(m.as_str()), + + ReplacementPart::Group(_) => { + panic!("unexpected Regex group replacement") + } + } + } + result + } } #[derive(Debug, Default)] @@ -352,7 +370,7 @@ mod tests { let template = ReplacementTemplate::default(); let caps = caps_for("foo", "foo"); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, ""); } @@ -362,7 +380,7 @@ mod tests { let template = ReplacementTemplate::new(vec![ReplacementPart::Literal("hello".into())]); let caps = caps_for("abc", "abc"); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "hello"); } @@ -375,7 +393,7 @@ mod tests { ]); let caps = caps_for(r"foo\d+", "foo42"); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "got: foo42"); } @@ -388,7 +406,7 @@ mod tests { ]); let caps = caps_for(r"foo(\d+)", "foo42"); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "number: 42"); } @@ -403,7 +421,7 @@ mod tests { ]); let caps = caps_for(r"(\w+):(\d+)", "x:123"); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "key: x, value: 123"); } @@ -418,7 +436,7 @@ mod tests { ]); let caps = caps_for(r"(\w+):(\d+)", "x:123"); - let result = template.apply(&caps); + let result = template.apply_captures(&caps); assert!(result.is_err()); let msg = result.unwrap_err().to_string(); diff --git a/src/uu/sed/src/processor.rs b/src/uu/sed/src/processor.rs index bd2522a..41dfc0a 100644 --- a/src/uu/sed/src/processor.rs +++ b/src/uu/sed/src/processor.rs @@ -201,34 +201,55 @@ fn substitute( let regex = re_or_saved_re(&sub.regex, context)?; - for caps in regex.captures_iter(text) { - count += 1; - let caps = caps.map_err(|e| { - USimpleError::new( - 2, - format!("regular expression capture retrieval error: {}", e), - ) - })?; + match (sub.occurrence, sub.replacement.max_group_number) { + (1, 0) => { + let m = regex.find(text); + let m = m.map_err(|e| { + USimpleError::new( + 2, + format!("regular expression match retrieval error: {}", e), + ) + })?; + if let Some(m) = m { + result.push_str(&text[last_end..m.start()]); - let m = caps.get(0).unwrap(); - - // Always write the unmatched text before this match. - result.push_str(&text[last_end..m.start()]); - - if sub.occurrence == 0 || count == sub.occurrence { - let replacement = sub.replacement.apply(&caps)?; - result.push_str(&replacement); - replaced = true; - } else { - // Not the target match — leave the match unchanged. - result.push_str(m.as_str()); + let replacement = sub.replacement.apply_match(&m); + result.push_str(&replacement); + replaced = true; + last_end = m.end(); + } } + (_, _) => { + for caps in regex.captures_iter(text) { + count += 1; + let caps = caps.map_err(|e| { + USimpleError::new( + 2, + format!("regular expression capture retrieval error: {}", e), + ) + })?; - last_end = m.end(); + let m = caps.get(0).unwrap(); - // Early exit if only a specific occurrence (likely 1) needed replacing. - if count == sub.occurrence { - break; + // Always write the unmatched text before this match. + result.push_str(&text[last_end..m.start()]); + + if sub.occurrence == 0 || count == sub.occurrence { + let replacement = sub.replacement.apply_captures(&caps)?; + result.push_str(&replacement); + replaced = true; + } else { + // Not the target match — leave the match unchanged. + result.push_str(m.as_str()); + } + + last_end = m.end(); + + // Early exit if only a specific occurrence (likely 1) needed replacing. + if count == sub.occurrence { + break; + } + } } } From a6010ae54be658d5b400935a3b279b6dca31864a Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sat, 24 May 2025 22:54:18 +0300 Subject: [PATCH 09/17] Revert "Try to specialize regex retrieval" This reverts commit f18f772f31b21660df54474dc3f0872ce53c8612. No performance improvement was seen. --- src/uu/sed/src/command.rs | 36 +++++-------------- src/uu/sed/src/processor.rs | 69 +++++++++++++------------------------ 2 files changed, 33 insertions(+), 72 deletions(-) diff --git a/src/uu/sed/src/command.rs b/src/uu/sed/src/command.rs index e2fb162..6e06f30 100644 --- a/src/uu/sed/src/command.rs +++ b/src/uu/sed/src/command.rs @@ -13,7 +13,7 @@ use crate::named_writer::NamedWriter; -use fancy_regex::{Captures, Match, Regex}; +use fancy_regex::{Captures, Regex}; use std::borrow::Cow; use std::cell::RefCell; use std::collections::HashMap; @@ -155,9 +155,9 @@ impl ReplacementTemplate { /// Apply the template to the given RE captures. /// Example: /// let result = regex.replace_all(input, |caps: &Captures| { - /// template.apply_captures(caps) }); + /// template.apply(caps) }); /// Returns an error if a backreference in the template was not matched by the RE. - pub fn apply_captures(&self, caps: &Captures) -> UResult { + pub fn apply(&self, caps: &Captures) -> UResult { let mut result = String::new(); // Invalid group numbers may end here through (unkown at compile time) @@ -191,24 +191,6 @@ impl ReplacementTemplate { Ok(result) } - - /// Apply the template to the given RE single match. - pub fn apply_match(&self, m: &Match) -> String { - let mut result = String::new(); - - for part in &self.parts { - match part { - ReplacementPart::Literal(s) => result.push_str(s), - - ReplacementPart::WholeMatch => result.push_str(m.as_str()), - - ReplacementPart::Group(_) => { - panic!("unexpected Regex group replacement") - } - } - } - result - } } #[derive(Debug, Default)] @@ -370,7 +352,7 @@ mod tests { let template = ReplacementTemplate::default(); let caps = caps_for("foo", "foo"); - let result = template.apply_captures(&caps).unwrap(); + let result = template.apply(&caps).unwrap(); assert_eq!(result, ""); } @@ -380,7 +362,7 @@ mod tests { let template = ReplacementTemplate::new(vec![ReplacementPart::Literal("hello".into())]); let caps = caps_for("abc", "abc"); - let result = template.apply_captures(&caps).unwrap(); + let result = template.apply(&caps).unwrap(); assert_eq!(result, "hello"); } @@ -393,7 +375,7 @@ mod tests { ]); let caps = caps_for(r"foo\d+", "foo42"); - let result = template.apply_captures(&caps).unwrap(); + let result = template.apply(&caps).unwrap(); assert_eq!(result, "got: foo42"); } @@ -406,7 +388,7 @@ mod tests { ]); let caps = caps_for(r"foo(\d+)", "foo42"); - let result = template.apply_captures(&caps).unwrap(); + let result = template.apply(&caps).unwrap(); assert_eq!(result, "number: 42"); } @@ -421,7 +403,7 @@ mod tests { ]); let caps = caps_for(r"(\w+):(\d+)", "x:123"); - let result = template.apply_captures(&caps).unwrap(); + let result = template.apply(&caps).unwrap(); assert_eq!(result, "key: x, value: 123"); } @@ -436,7 +418,7 @@ mod tests { ]); let caps = caps_for(r"(\w+):(\d+)", "x:123"); - let result = template.apply_captures(&caps); + let result = template.apply(&caps); assert!(result.is_err()); let msg = result.unwrap_err().to_string(); diff --git a/src/uu/sed/src/processor.rs b/src/uu/sed/src/processor.rs index 41dfc0a..bd2522a 100644 --- a/src/uu/sed/src/processor.rs +++ b/src/uu/sed/src/processor.rs @@ -201,55 +201,34 @@ fn substitute( let regex = re_or_saved_re(&sub.regex, context)?; - match (sub.occurrence, sub.replacement.max_group_number) { - (1, 0) => { - let m = regex.find(text); - let m = m.map_err(|e| { - USimpleError::new( - 2, - format!("regular expression match retrieval error: {}", e), - ) - })?; - if let Some(m) = m { - result.push_str(&text[last_end..m.start()]); + for caps in regex.captures_iter(text) { + count += 1; + let caps = caps.map_err(|e| { + USimpleError::new( + 2, + format!("regular expression capture retrieval error: {}", e), + ) + })?; - let replacement = sub.replacement.apply_match(&m); - result.push_str(&replacement); - replaced = true; - last_end = m.end(); - } + let m = caps.get(0).unwrap(); + + // Always write the unmatched text before this match. + result.push_str(&text[last_end..m.start()]); + + if sub.occurrence == 0 || count == sub.occurrence { + let replacement = sub.replacement.apply(&caps)?; + result.push_str(&replacement); + replaced = true; + } else { + // Not the target match — leave the match unchanged. + result.push_str(m.as_str()); } - (_, _) => { - for caps in regex.captures_iter(text) { - count += 1; - let caps = caps.map_err(|e| { - USimpleError::new( - 2, - format!("regular expression capture retrieval error: {}", e), - ) - })?; - let m = caps.get(0).unwrap(); + last_end = m.end(); - // Always write the unmatched text before this match. - result.push_str(&text[last_end..m.start()]); - - if sub.occurrence == 0 || count == sub.occurrence { - let replacement = sub.replacement.apply_captures(&caps)?; - result.push_str(&replacement); - replaced = true; - } else { - // Not the target match — leave the match unchanged. - result.push_str(m.as_str()); - } - - last_end = m.end(); - - // Early exit if only a specific occurrence (likely 1) needed replacing. - if count == sub.occurrence { - break; - } - } + // Early exit if only a specific occurrence (likely 1) needed replacing. + if count == sub.occurrence { + break; } } From 5fad204a09b72a82fa30af59ac93fed8bdb7024a Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sun, 25 May 2025 11:19:02 +0300 Subject: [PATCH 10/17] Use regex::bytes whenever possible This improves one benchmark case by 20%: access-log-all-del rust-sed is 1.2 times faster than rust-sed-f18f772 Unfortunatelly, all other cases remain unaffected. --- Cargo.lock | 2 + Cargo.toml | 2 + src/uu/sed/Cargo.toml | 1 + src/uu/sed/src/command.rs | 33 ++-- src/uu/sed/src/compiler.rs | 39 ++--- src/uu/sed/src/fast_io.rs | 48 ++++-- src/uu/sed/src/fast_regex.rs | 292 +++++++++++++++++++++++++++++++++++ src/uu/sed/src/processor.rs | 29 ++-- src/uu/sed/src/sed.rs | 1 + 9 files changed, 384 insertions(+), 63 deletions(-) create mode 100644 src/uu/sed/src/fast_regex.rs diff --git a/Cargo.lock b/Cargo.lock index 15c8473..10be09f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -705,6 +705,7 @@ dependencies = [ "libc", "memchr", "memmap2", + "once_cell", "phf", "phf_codegen", "pretty_assertions", @@ -899,6 +900,7 @@ dependencies = [ "fancy-regex", "memchr", "memmap2", + "once_cell", "regex", "tempfile", "uucore", diff --git a/Cargo.toml b/Cargo.toml index 61a7d8c..ca18467 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ fancy-regex = "0.14.0" libc = "0.2.153" memchr = "2.7.4" memmap2 = "0.9" +once_cell = "1.21.3" phf = "0.11.2" phf_codegen = "0.11.2" rand = { version = "0.9", features = ["small_rng"] } @@ -59,6 +60,7 @@ ctor = "0.4.1" fancy-regex = { workspace = true } memchr = { workspace = true } memmap2.workspace = true +once_cell = { workspace = true } phf = { workspace = true } sed = { optional = true, version = "0.0.1", package = "uu_sed", path = "src/uu/sed" } sysinfo = { workspace = true } diff --git a/src/uu/sed/Cargo.toml b/src/uu/sed/Cargo.toml index 8487ab2..acf2718 100644 --- a/src/uu/sed/Cargo.toml +++ b/src/uu/sed/Cargo.toml @@ -19,6 +19,7 @@ memchr = { workspace = true } regex = { workspace = true } tempfile = { workspace = true } memmap2 = { workspace = true } +once_cell = { workspace = true } uucore = { workspace = true } [lib] diff --git a/src/uu/sed/src/command.rs b/src/uu/sed/src/command.rs index 6e06f30..8f8dccd 100644 --- a/src/uu/sed/src/command.rs +++ b/src/uu/sed/src/command.rs @@ -11,9 +11,9 @@ // TODO: remove when compile is implemented #![allow(dead_code)] +use crate::fast_regex::{Captures, Regex}; use crate::named_writer::NamedWriter; -use fancy_regex::{Captures, Regex}; use std::borrow::Cow; use std::cell::RefCell; use std::collections::HashMap; @@ -177,14 +177,12 @@ impl ReplacementTemplate { ReplacementPart::Literal(s) => result.push_str(s), ReplacementPart::WholeMatch => { - result.push_str(caps.get(0).map_or("", |m| m.as_str())); + result.push_str(caps.get(0)?.map(|m| m.as_str()).unwrap_or("")); } ReplacementPart::Group(n) => { - result.push_str( - caps.get((*n).try_into().unwrap()) - .map_or("", |m| m.as_str()), - ); + let i: usize = (*n).try_into().unwrap(); + result.push_str(caps.get(i)?.map(|m| m.as_str()).unwrap_or("")); } } } @@ -336,12 +334,13 @@ pub struct InputAction { #[cfg(test)] mod tests { use super::*; + use crate::fast_io::IOChunk; // Return the captures for the RE applied to the specified string - fn caps_for<'a>(re: &str, input: &'a str) -> Captures<'a> { + fn caps_for<'a>(re: &str, chunk: &'a mut IOChunk) -> Captures<'a> { Regex::new(re) .unwrap() - .captures(input) + .captures(chunk) .unwrap() .expect("captures") } @@ -350,7 +349,8 @@ mod tests { // s/foo// fn test_empty_template() { let template = ReplacementTemplate::default(); - let caps = caps_for("foo", "foo"); + let input = &mut IOChunk::from_str("foo"); + let caps = caps_for("foo", input); let result = template.apply(&caps).unwrap(); assert_eq!(result, ""); @@ -360,7 +360,8 @@ mod tests { // s/abc/hello/ fn test_literal_only() { let template = ReplacementTemplate::new(vec![ReplacementPart::Literal("hello".into())]); - let caps = caps_for("abc", "abc"); + let input = &mut IOChunk::from_str("abc"); + let caps = caps_for("abc", input); let result = template.apply(&caps).unwrap(); assert_eq!(result, "hello"); @@ -373,7 +374,8 @@ mod tests { ReplacementPart::Literal("got: ".into()), ReplacementPart::WholeMatch, ]); - let caps = caps_for(r"foo\d+", "foo42"); + let input = &mut IOChunk::from_str("foo42"); + let caps = caps_for(r"foo\d+", input); let result = template.apply(&caps).unwrap(); assert_eq!(result, "got: foo42"); @@ -386,7 +388,8 @@ mod tests { ReplacementPart::Literal("number: ".into()), ReplacementPart::Group(1), ]); - let caps = caps_for(r"foo(\d+)", "foo42"); + let input = &mut IOChunk::from_str("foo42"); + let caps = caps_for(r"foo(\d+)", input); let result = template.apply(&caps).unwrap(); assert_eq!(result, "number: 42"); @@ -401,7 +404,8 @@ mod tests { ReplacementPart::Literal(", value: ".into()), ReplacementPart::Group(2), ]); - let caps = caps_for(r"(\w+):(\d+)", "x:123"); + let input = &mut IOChunk::from_str("x:123"); + let caps = caps_for(r"(\w+):(\d+)", input); let result = template.apply(&caps).unwrap(); assert_eq!(result, "key: x, value: 123"); @@ -416,7 +420,8 @@ mod tests { ReplacementPart::Literal(", value: ".into()), ReplacementPart::Group(3), ]); - let caps = caps_for(r"(\w+):(\d+)", "x:123"); + let input = &mut IOChunk::from_str("x:123"); + let caps = caps_for(r"(\w+):(\d+)", input); let result = template.apply(&caps); assert!(result.is_err()); diff --git a/src/uu/sed/src/compiler.rs b/src/uu/sed/src/compiler.rs index 5fdcfc3..d3a7c28 100644 --- a/src/uu/sed/src/compiler.rs +++ b/src/uu/sed/src/compiler.rs @@ -15,10 +15,10 @@ use crate::command::{ use crate::delimited_parser::{ compilation_error, parse_char_escape, parse_regex, parse_transliteration, }; +use crate::fast_regex::Regex; use crate::named_writer::NamedWriter; use crate::script_char_provider::ScriptCharProvider; use crate::script_line_provider::ScriptLineProvider; -use fancy_regex::Regex; use std::borrow::Cow; use std::cell::RefCell; use std::collections::HashMap; @@ -1167,6 +1167,7 @@ fn lookup_command(cmd: char) -> Option<&'static CommandSpec> { #[cfg(test)] mod tests { use super::*; + use crate::fast_io::IOChunk; // Return an empty line provider and a char provider for the specified str. fn make_providers(input: &str) -> (ScriptLineProvider, ScriptCharProvider) { @@ -1384,8 +1385,8 @@ mod tests { let regex = compile_regex(&lines, &chars, "abc", &ctx(), false) .unwrap() .expect("regex should be present"); - assert!(regex.is_match("abc").unwrap()); - assert!(!regex.is_match("ABC").unwrap()); + assert!(regex.is_match(&mut IOChunk::from_str("abc")).unwrap()); + assert!(!regex.is_match(&mut IOChunk::from_str("ABC")).unwrap()); } #[test] @@ -1394,9 +1395,9 @@ mod tests { let regex = compile_regex(&lines, &chars, "abc", &ctx(), true) .unwrap() .expect("regex should be present"); - assert!(regex.is_match("abc").unwrap()); - assert!(regex.is_match("ABC").unwrap()); - assert!(regex.is_match("AbC").unwrap()); + assert!(regex.is_match(&mut IOChunk::from_str("abc")).unwrap()); + assert!(regex.is_match(&mut IOChunk::from_str("ABC")).unwrap()); + assert!(regex.is_match(&mut IOChunk::from_str("AbC")).unwrap()); } #[test] @@ -1444,7 +1445,7 @@ mod tests { let addr = compile_address(&lines, &mut chars, &ctx()).unwrap(); assert!(matches!(addr.atype, AddressType::Re)); if let AddressValue::Regex(Some(re)) = addr.value { - assert!(re.is_match("hello").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("hello")).unwrap()); } else { panic!("expected Regex address value"); } @@ -1456,7 +1457,7 @@ mod tests { let addr = compile_address(&lines, &mut chars, &ctx()).unwrap(); assert!(matches!(addr.atype, AddressType::Re)); if let AddressValue::Regex(Some(re)) = addr.value { - assert!(re.is_match("hello").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("hello")).unwrap()); } else { panic!("expected Regex address value"); } @@ -1468,7 +1469,7 @@ mod tests { let addr = compile_address(&lines, &mut chars, &ctx()).unwrap(); assert!(matches!(addr.atype, AddressType::Re)); if let AddressValue::Regex(Some(re)) = addr.value { - assert!(!re.is_match("helio").unwrap()); + assert!(!re.is_match(&mut IOChunk::from_str("helio")).unwrap()); } else { panic!("expected Regex address value"); } @@ -1480,7 +1481,7 @@ mod tests { let addr = compile_address(&lines, &mut chars, &ctx()).unwrap(); assert!(matches!(addr.atype, AddressType::Re)); if let AddressValue::Regex(Some(re)) = addr.value { - assert!(re.is_match("hello").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("hello")).unwrap()); } else { panic!("expected Regex address value"); } @@ -1492,7 +1493,7 @@ mod tests { let addr = compile_address(&lines, &mut chars, &ctx()).unwrap(); assert!(matches!(addr.atype, AddressType::Re)); if let AddressValue::Regex(Some(re)) = addr.value { - assert!(re.is_match("HELLO").unwrap()); // case-insensitive + assert!(re.is_match(&mut IOChunk::from_str("HELLO")).unwrap()); // case-insensitive } else { panic!("expected Regex address value"); } @@ -1583,8 +1584,8 @@ mod tests { AddressType::Re )); if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr1.as_ref().unwrap().value { - assert!(re.is_match("foo").unwrap()); - assert!(!re.is_match("bar").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("foo")).unwrap()); + assert!(!re.is_match(&mut IOChunk::from_str("bar")).unwrap()); } else { panic!("expected a regex address"); }; @@ -1603,8 +1604,8 @@ mod tests { AddressType::Re )); if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr1.as_ref().unwrap().value { - assert!(re.is_match("foo").unwrap()); - assert!(!re.is_match("bar").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("foo")).unwrap()); + assert!(!re.is_match(&mut IOChunk::from_str("bar")).unwrap()); } else { panic!("expected a regex address"); } @@ -1614,8 +1615,8 @@ mod tests { AddressType::Re )); if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr2.as_ref().unwrap().value { - assert!(re.is_match("bar").unwrap()); - assert!(!re.is_match("foo").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("bar")).unwrap()); + assert!(!re.is_match(&mut IOChunk::from_str("foo")).unwrap()); } else { panic!("expected a regex address"); }; @@ -1633,8 +1634,8 @@ mod tests { AddressType::Re )); if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr1.as_ref().unwrap().value { - assert!(re.is_match("FOO").unwrap()); - assert!(re.is_match("foo").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("FOO")).unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("foo")).unwrap()); } else { panic!("expected a regex address with case-insensitive match"); }; diff --git a/src/uu/sed/src/fast_io.rs b/src/uu/sed/src/fast_io.rs index 1cc7e12..2bc678d 100644 --- a/src/uu/sed/src/fast_io.rs +++ b/src/uu/sed/src/fast_io.rs @@ -18,6 +18,7 @@ use memchr::memchr; #[cfg(unix)] use memmap2::Mmap; +use std::cell::Cell; use std::fs::File; use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; @@ -138,7 +139,7 @@ impl ReadLineCursor { /// As chunk of data that is input and can be output, often very efficiently #[derive(Debug, PartialEq, Eq)] pub struct IOChunk<'a> { - utf8_verified: bool, // True if the contents are valid UTF-8 + utf8_verified: Cell, // True if the contents are valid UTF-8 content: IOChunkContent<'a>, } @@ -146,14 +147,14 @@ impl<'a> IOChunk<'a> { /// Construct an IOChunk from the given content fn from_content(content: IOChunkContent<'a>) -> Self { Self { - utf8_verified: false, + utf8_verified: Cell::new(false), content, } } /// Clear the object's contents, converting it into Owned if needed. pub fn clear(&mut self) { - self.utf8_verified = true; + self.utf8_verified.set(true); match &mut self.content { IOChunkContent::Owned { content, @@ -185,10 +186,19 @@ impl<'a> IOChunk<'a> { } } + #[cfg(test)] + /// Create an Owned newline-terminated IOChunk from a string. + pub fn from_str(s: &str) -> Self { + IOChunk { + content: IOChunkContent::new_owned(s.to_string(), true), + utf8_verified: Cell::new(false), + } + } + /// Set the object's contents to the specified string. /// Convert it into Owned if needed. pub fn set_to_string(&mut self, new_content: String, add_newline: bool) { - self.utf8_verified = true; + self.utf8_verified.set(true); match &mut self.content { IOChunkContent::Owned { content, @@ -206,16 +216,16 @@ impl<'a> IOChunk<'a> { } /// Return the content as a str. - pub fn as_str(&mut self) -> Result<&str, Box> { + pub fn as_str(&self) -> Result<&str, Box> { match &self.content { #[cfg(unix)] IOChunkContent::MmapInput { content, .. } => { - if self.utf8_verified { + if self.utf8_verified.get() { // Use cached result Ok(unsafe { self.content.as_str_unchecked() }) } else { let result = str::from_utf8(content); - self.utf8_verified = true; + self.utf8_verified.set(true); result.map_err(|e| USimpleError::new(1, e.to_string())) } } @@ -223,6 +233,15 @@ impl<'a> IOChunk<'a> { } } + /// Return the raw byte content (always safe). + pub fn as_bytes(&self) -> &[u8] { + match &self.content { + #[cfg(unix)] + IOChunkContent::MmapInput { content, .. } => content, + IOChunkContent::Owned { content, .. } => content.as_bytes(), + } + } + /// Convert content to the Owned variant if it's not already. /// Fails if the conversion to UTF-8 fails. pub fn ensure_owned(&mut self) -> Result<(), Box> { @@ -235,7 +254,7 @@ impl<'a> IOChunk<'a> { let has_newline = full_span.last().copied() == Some(b'\n'); self.content = IOChunkContent::new_owned(valid_str.to_string(), has_newline); - self.utf8_verified = true; + self.utf8_verified.set(true); Ok(()) } Err(e) => Err(USimpleError::new(1, e.to_string())), @@ -937,7 +956,7 @@ mod tests { { assert_eq!(content, "first line"); assert!(has_newline); - assert!(!utf8_verified); + assert!(!utf8_verified.get()); assert!(!last_line); } else { panic!("Expected IOChunkContent::Owned"); @@ -963,7 +982,7 @@ mod tests { panic!("Expected IOChunkContent::Owned"); } - if let Some((mut content, last_line)) = reader.get_line()? { + if let Some((content, last_line)) = reader.get_line()? { assert_eq!(content.as_str().unwrap(), "last line"); assert!(last_line); } else { @@ -1001,7 +1020,7 @@ mod tests { { assert_eq!(content, b"first line"); assert_eq!(full_span, b"first line\n"); - assert!(!utf8_verified); + assert!(!utf8_verified.get()); assert!(!last_line); } else { panic!("Expected IOChunkContent::MapInput"); @@ -1021,15 +1040,16 @@ mod tests { { assert_eq!(content, b"second line"); assert_eq!(full_span, b"second line\n"); - assert!(!utf8_verified); + assert!(!utf8_verified.get()); assert!(!last_line); } else { panic!("Expected IOChunkContent::MapInput"); } - if let Some((mut content, last_line)) = reader.get_line()? { + if let Some((content, last_line)) = reader.get_line()? { + assert_eq!(content.as_bytes(), b"last line"); assert_eq!(content.as_str().unwrap(), "last line"); - assert!(content.utf8_verified); + assert!(content.utf8_verified.get()); assert!(last_line); // Cached version assert_eq!(content.as_str().unwrap(), "last line"); diff --git a/src/uu/sed/src/fast_regex.rs b/src/uu/sed/src/fast_regex.rs new file mode 100644 index 0000000..3375bd6 --- /dev/null +++ b/src/uu/sed/src/fast_regex.rs @@ -0,0 +1,292 @@ +// A unified interface to byte and fancy Regex +// +// This allows using byte Regex when possible, resorting to the +// slower fancy_regex crate when needed. +// +// SPDX-License-Identifier: MIT +// Copyright (c) 2025 Diomidis Spinellis +// +// This file is part of the uutils sed package. +// It is licensed under the MIT License. +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use fancy_regex::{ + CaptureMatches as FancyCaptureMatches, Captures as FancyCaptures, Regex as FancyRegex, +}; +use once_cell::sync::Lazy; +use regex::bytes::{ + CaptureMatches as ByteCaptureMatches, Captures as ByteCaptures, Regex as ByteRegex, +}; +use std::error::Error; +use uucore::error::{UResult, USimpleError}; + +use crate::fast_io::IOChunk; + +#[derive(Clone, Debug)] +/// A regular expression that can be implemented through Byte or Fancy-Regex +// This enables a fast bytes path where possible. +pub enum Regex { + Byte(ByteRegex), + Fancy(FancyRegex), +} + +impl Regex { + /// Construct the most efficient engine possible + pub fn new(pattern: &str) -> Result> { + static NEEDS_FANCY_RE: Lazy = Lazy::new(|| { + regex::Regex::new( + r"(?x) # Turn on verbose mode + ( # An ASCII-incompatible RE + ( ^ | [^\\] ) # Non-escaped, so BOL or not \ + ( # A potentially incompatible match + \. # . matches any Unicode character + | \[\^ # Bracketed negative character class + | \(\?i # Case insensitive is done on Unicode + | \\[WwDdSsBbPp] # Unicode classes + | \\[0-9] # Only fancy supports back-references + ) + ) + | [^\x01-\x7f] # Or any non-ASCII character in the RE + ", + ) + .unwrap() + }); + + if NEEDS_FANCY_RE.is_match(pattern) { + Ok(Self::Fancy(FancyRegex::new(pattern)?)) + } else { + Ok(Self::Byte(ByteRegex::new(pattern)?)) + } + } + + /// Return true if this is a byte-based regex. + pub fn is_byte(&self) -> bool { + matches!(self, Regex::Byte(_)) + } + + /// Check if the regex matches the content of the IOChunk. + pub fn is_match(&self, chunk: &mut IOChunk) -> UResult { + match self { + Regex::Byte(re) => Ok(re.is_match(chunk.as_bytes())), + Regex::Fancy(re) => { + let text = chunk.as_str()?; + re.is_match(text) + .map_err(|e| USimpleError::new(2, e.to_string())) + } + } + } + + /// Return an iterator over capture groups. + pub fn captures_iter<'t>(&'t self, chunk: &'t IOChunk) -> UResult> { + match self { + Regex::Byte(re) => Ok(CaptureMatches::Byte(re.captures_iter(chunk.as_bytes()))), + Regex::Fancy(re) => { + let text = chunk.as_str()?; + Ok(CaptureMatches::Fancy(re.captures_iter(text))) + } + } + } + + /// Return the number of capture groups, including group 0. + pub fn captures_len(&self) -> usize { + match self { + Regex::Byte(re) => re.captures_len(), + Regex::Fancy(re) => re.captures_len(), + } + } + + /// Return the elements of the first capture. + pub fn captures<'t>(&self, chunk: &'t IOChunk) -> UResult>> { + match self { + Regex::Byte(re) => { + let bytes = chunk.as_bytes(); + Ok(re.captures(bytes).map(Captures::Byte)) + } + + Regex::Fancy(re) => { + let text = chunk.as_str()?; + match re.captures(text) { + Ok(Some(caps)) => Ok(Some(Captures::Fancy(caps))), + Ok(None) => Ok(None), + Err(e) => Err(USimpleError::new(2, e.to_string())), + } + } + } + } +} + +/// Unified enum for holding either byte or fancy capture iterators. +pub enum CaptureMatches<'t> { + Byte(ByteCaptureMatches<'t, 't>), + Fancy(FancyCaptureMatches<'t, 't>), +} + +impl<'t> Iterator for CaptureMatches<'t> { + type Item = UResult>; + + fn next(&mut self) -> Option { + match self { + CaptureMatches::Byte(iter) => iter.next().map(|caps| Ok(Captures::Byte(caps))), + CaptureMatches::Fancy(iter) => match iter.next() { + Some(Ok(caps)) => Some(Ok(Captures::Fancy(caps))), + Some(Err(e)) => Some(Err(USimpleError::new( + 2, + format!("error retrieving RE captures: {}", e), + ))), + None => None, + }, + } + } +} + +#[derive(Debug)] +/// Result type for RE capture get(n) +pub struct Match<'t> { + start: usize, // Match start + end: usize, // Match end + text: &'t str, // Actual match +} + +/// Provide interface compatible with Regex::Match. +impl<'t> Match<'t> { + pub fn start(&self) -> usize { + self.start + } + + pub fn end(&self) -> usize { + self.end + } + + pub fn as_str(&self) -> &'t str { + self.text + } +} + +/// Provide interface compatible with Regex::Captures. +pub enum Captures<'t> { + Byte(ByteCaptures<'t>), + Fancy(FancyCaptures<'t>), +} + +impl<'t> Captures<'t> { + /// Get capture group at index `i` + /// Returns Ok(None) if the group didn't match. + /// Returns Err if UTF-8 conversion fails (in Byte variant). + pub fn get(&self, i: usize) -> UResult>> { + match self { + Captures::Byte(caps) => match caps.get(i) { + Some(m) => Ok(Some(Match { + start: m.start(), + end: m.end(), + text: std::str::from_utf8(m.as_bytes()) + .map_err(|e| USimpleError::new(1, e.to_string()))?, + })), + None => Ok(None), + }, + Captures::Fancy(caps) => match caps.get(i) { + Some(m) => Ok(Some(Match { + start: m.start(), + end: m.end(), + text: m.as_str(), + })), + None => Ok(None), + }, + } + } + + /// Return the number of capture groups (including group 0). + pub fn len(&self) -> usize { + match self { + Captures::Byte(caps) => caps.len(), + Captures::Fancy(caps) => caps.len(), + } + } + + /// Return true if there are no captures. + // Unused, but provided for completeness. + pub fn is_empty(&self) -> bool { + match self { + Captures::Byte(caps) => caps.len() == 0, + Captures::Fancy(caps) => caps.len() == 0, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_byte(pattern: &str) { + let re = Regex::new(pattern).unwrap(); + assert!(re.is_byte(), "Pattern {:?} should use Byte engine", pattern); + } + + fn assert_fancy(pattern: &str) { + let re = Regex::new(pattern).unwrap(); + assert!( + !re.is_byte(), + "Pattern {:?} should use Fancy engine", + pattern + ); + } + + #[test] + fn selects_byte_regex_for_simple_ascii() { + assert_byte(r"foo"); + assert_byte(r"foo|bar"); + assert_byte(r"^foo[0-9]+bar$"); + // Escaped Unicode triggers shouldn't trigger RE. + assert_byte(r"\."); + assert_byte(r"\[^x]"); + assert_byte(r"\\(?i)"); + assert_byte(r"\\w"); + } + + #[test] + fn selects_fancy_for_unicode_class_bol() { + assert_fancy(r"\p{L}+"); // Unicode letter class + assert_fancy(r"\W"); // \W is Unicode-aware. + assert_fancy(r"\S+"); // \S is Unicode-aware. + assert_fancy(r"\d"); // \d includes all Unicode digits. + } + + #[test] + fn selects_fancy_for_unicode_class_non_bol() { + assert_fancy(r"x\p{L}+"); // Unicode letter class + assert_fancy(r"x\W"); // \W is Unicode-aware. + assert_fancy(r"x\S+"); // \S is Unicode-aware. + assert_fancy(r"x\d"); // \d includes all Unicode digits. + } + + #[test] + fn selects_fancy_for_dot() { + assert_fancy(r"."); // Dot matches any Unicode char. + } + + #[test] + fn selects_fancy_for_inline_flags() { + assert_fancy(r"(?i)abc"); // Unicode case-insensitive + } + + #[test] + fn selects_fancy_for_backrefs() { + assert_fancy(r"(\w+):\1"); // back-reference \1 + } + + #[test] + fn selects_fancy_for_non_ascii_literals() { + assert_fancy("naïve"); // Contains literal non-ASCII. + assert_fancy("café"); // Contains literal non-ASCII. + } + + #[test] + fn handles_invalid_regex_gracefully() { + let err = Regex::new("(").unwrap_err().to_string(); + assert!( + err.contains("unclosed group") || err.contains("error parsing"), + "Unexpected error: {}", + err + ); + } +} diff --git a/src/uu/sed/src/processor.rs b/src/uu/sed/src/processor.rs index bd2522a..83a574b 100644 --- a/src/uu/sed/src/processor.rs +++ b/src/uu/sed/src/processor.rs @@ -13,9 +13,10 @@ use crate::command::{ ProcessingContext, Substitution, Transliteration, }; use crate::fast_io::{IOChunk, LineReader, OutputBuffer}; +use crate::fast_regex::Regex; use crate::in_place::InPlace; use crate::named_writer; -use fancy_regex::Regex; + use std::cell::RefCell; use std::io::{self, IsTerminal}; use std::path::PathBuf; @@ -32,10 +33,7 @@ fn match_address( AddressType::Re => { if let AddressValue::Regex(ref re) = addr.value { let regex = re_or_saved_re(re, context)?; - let matched = regex.is_match(pattern.as_str()?).map_err(|e| { - USimpleError::new(2, format!("regular expression match error: {}", e)) - })?; - Ok(matched) + regex.is_match(pattern) } else { Ok(false) } @@ -197,23 +195,22 @@ fn substitute( let mut result = String::new(); let mut replaced = false; - let text = pattern.as_str()?; + let mut text: Option<&str> = None; let regex = re_or_saved_re(&sub.regex, context)?; - for caps in regex.captures_iter(text) { + // Iterate over multiple captures of the RE in the pattern. + for caps in regex.captures_iter(pattern)? { count += 1; - let caps = caps.map_err(|e| { - USimpleError::new( - 2, - format!("regular expression capture retrieval error: {}", e), - ) - })?; + let caps = caps?; - let m = caps.get(0).unwrap(); + let m = caps.get(0)?.unwrap(); // Always write the unmatched text before this match. - result.push_str(&text[last_end..m.start()]); + if text.is_none() { + text = Some(pattern.as_str()?); + } + result.push_str(&text.unwrap()[last_end..m.start()]); if sub.occurrence == 0 || count == sub.occurrence { let replacement = sub.replacement.apply(&caps)?; @@ -234,7 +231,7 @@ fn substitute( // Handle substitution success. if replaced { - result.push_str(&text[last_end..]); + result.push_str(&text.unwrap()[last_end..]); pattern.set_to_string(result, pattern.is_newline_terminated()); diff --git a/src/uu/sed/src/sed.rs b/src/uu/sed/src/sed.rs index 13f3a1b..d61bc96 100644 --- a/src/uu/sed/src/sed.rs +++ b/src/uu/sed/src/sed.rs @@ -12,6 +12,7 @@ pub mod command; pub mod compiler; pub mod delimited_parser; pub mod fast_io; +pub mod fast_regex; pub mod in_place; pub mod named_writer; pub mod processor; From 90d60b87a69b3d4107b25184fafc9ea5fc3cbd09 Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sat, 24 May 2025 22:51:35 +0300 Subject: [PATCH 11/17] Try again to specialize regex retrieval The hope is that the now available regex::bytes will run find() faster than captures_iter(). Indeed, using find() increases the performance of several benchmark cases by 2-12%. no-op-short rust-sed is 1.02 times faster than rust-sed-5fad204 access-log-no-op rust-sed is 1.01 times faster than rust-sed-5fad204 access-log-no-subst rust-sed is 1.08 times faster than rust-sed-5fad204 access-log-subst rust-sed is 1.12 times faster than rust-sed-5fad204 access-log-translit rust-sed is 1.02 times faster than rust-sed-5fad204 access-log-complex-sub rust-sed is 1.04 times faster than rust-sed-5fad204 remove-cr rust-sed is 1.12 times faster than rust-sed-5fad204 genome-subst rust-sed is 1.02 times faster than rust-sed-5fad204 hanoi rust-sed is 1.01 times faster than rust-sed-5fad204 All other cases remain the same. --- src/uu/sed/src/command.rs | 36 ++++++++++++++++------ src/uu/sed/src/fast_regex.rs | 33 ++++++++++++++++++++ src/uu/sed/src/processor.rs | 60 +++++++++++++++++++++++------------- 3 files changed, 98 insertions(+), 31 deletions(-) diff --git a/src/uu/sed/src/command.rs b/src/uu/sed/src/command.rs index 8f8dccd..28e30e0 100644 --- a/src/uu/sed/src/command.rs +++ b/src/uu/sed/src/command.rs @@ -11,7 +11,7 @@ // TODO: remove when compile is implemented #![allow(dead_code)] -use crate::fast_regex::{Captures, Regex}; +use crate::fast_regex::{Captures, Match, Regex}; use crate::named_writer::NamedWriter; use std::borrow::Cow; @@ -155,9 +155,9 @@ impl ReplacementTemplate { /// Apply the template to the given RE captures. /// Example: /// let result = regex.replace_all(input, |caps: &Captures| { - /// template.apply(caps) }); + /// template.apply_captures(caps) }); /// Returns an error if a backreference in the template was not matched by the RE. - pub fn apply(&self, caps: &Captures) -> UResult { + pub fn apply_captures(&self, caps: &Captures) -> UResult { let mut result = String::new(); // Invalid group numbers may end here through (unkown at compile time) @@ -189,6 +189,24 @@ impl ReplacementTemplate { Ok(result) } + + /// Apply the template to the given RE single match. + pub fn apply_match(&self, m: &Match) -> String { + let mut result = String::new(); + + for part in &self.parts { + match part { + ReplacementPart::Literal(s) => result.push_str(s), + + ReplacementPart::WholeMatch => result.push_str(m.as_str()), + + ReplacementPart::Group(_) => { + panic!("unexpected Regex group replacement") + } + } + } + result + } } #[derive(Debug, Default)] @@ -352,7 +370,7 @@ mod tests { let input = &mut IOChunk::from_str("foo"); let caps = caps_for("foo", input); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, ""); } @@ -363,7 +381,7 @@ mod tests { let input = &mut IOChunk::from_str("abc"); let caps = caps_for("abc", input); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "hello"); } @@ -377,7 +395,7 @@ mod tests { let input = &mut IOChunk::from_str("foo42"); let caps = caps_for(r"foo\d+", input); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "got: foo42"); } @@ -391,7 +409,7 @@ mod tests { let input = &mut IOChunk::from_str("foo42"); let caps = caps_for(r"foo(\d+)", input); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "number: 42"); } @@ -407,7 +425,7 @@ mod tests { let input = &mut IOChunk::from_str("x:123"); let caps = caps_for(r"(\w+):(\d+)", input); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "key: x, value: 123"); } @@ -423,7 +441,7 @@ mod tests { let input = &mut IOChunk::from_str("x:123"); let caps = caps_for(r"(\w+):(\d+)", input); - let result = template.apply(&caps); + let result = template.apply_captures(&caps); assert!(result.is_err()); let msg = result.unwrap_err().to_string(); diff --git a/src/uu/sed/src/fast_regex.rs b/src/uu/sed/src/fast_regex.rs index 3375bd6..e8e52ed 100644 --- a/src/uu/sed/src/fast_regex.rs +++ b/src/uu/sed/src/fast_regex.rs @@ -114,6 +114,39 @@ impl Regex { } } } + + /// Return a non-capturing result for a single match. + pub fn find<'t>(&self, chunk: &'t IOChunk) -> UResult>> { + match self { + Regex::Byte(re) => { + let haystack = chunk.as_bytes(); + if let Some(m) = re.find(haystack) { + // Attempt UTF-8 decode for the match region only + let text = std::str::from_utf8(&haystack[m.start()..m.end()]) + .map_err(|e| USimpleError::new(2, e.to_string()))?; + Ok(Some(Match { + start: m.start(), + end: m.end(), + text, + })) + } else { + Ok(None) + } + } + Regex::Fancy(re) => { + let text = chunk.as_str()?; + match re.find(text) { + Ok(Some(m)) => Ok(Some(Match { + start: m.start(), + end: m.end(), + text: m.as_str(), + })), + Ok(None) => Ok(None), + Err(e) => Err(USimpleError::new(2, e.to_string())), + } + } + } + } } /// Unified enum for holding either byte or fancy capture iterators. diff --git a/src/uu/sed/src/processor.rs b/src/uu/sed/src/processor.rs index 83a574b..a9263ce 100644 --- a/src/uu/sed/src/processor.rs +++ b/src/uu/sed/src/processor.rs @@ -199,33 +199,49 @@ fn substitute( let regex = re_or_saved_re(&sub.regex, context)?; - // Iterate over multiple captures of the RE in the pattern. - for caps in regex.captures_iter(pattern)? { - count += 1; - let caps = caps?; + match (sub.occurrence, sub.replacement.max_group_number) { + (1, 0) => { + let m = regex.find(pattern)?; + if let Some(m) = m { + text = Some(pattern.as_str()?); + result.push_str(&text.unwrap()[last_end..m.start()]); - let m = caps.get(0)?.unwrap(); - - // Always write the unmatched text before this match. - if text.is_none() { - text = Some(pattern.as_str()?); + let replacement = sub.replacement.apply_match(&m); + result.push_str(&replacement); + replaced = true; + last_end = m.end(); + } } - result.push_str(&text.unwrap()[last_end..m.start()]); + (_, _) => { + // Iterate over multiple captures of the RE in the pattern. + for caps in regex.captures_iter(pattern)? { + count += 1; + let caps = caps?; - if sub.occurrence == 0 || count == sub.occurrence { - let replacement = sub.replacement.apply(&caps)?; - result.push_str(&replacement); - replaced = true; - } else { - // Not the target match — leave the match unchanged. - result.push_str(m.as_str()); - } + let m = caps.get(0)?.unwrap(); - last_end = m.end(); + // Always write the unmatched text before this match. + if text.is_none() { + text = Some(pattern.as_str()?); + } + result.push_str(&text.unwrap()[last_end..m.start()]); - // Early exit if only a specific occurrence (likely 1) needed replacing. - if count == sub.occurrence { - break; + if sub.occurrence == 0 || count == sub.occurrence { + let replacement = sub.replacement.apply_captures(&caps)?; + result.push_str(&replacement); + replaced = true; + } else { + // Not the target match — leave the match unchanged. + result.push_str(m.as_str()); + } + + last_end = m.end(); + + // Early exit if only a specific occurrence (likely 1) needed replacing. + if count == sub.occurrence { + break; + } + } } } From 28b5eb0b722e285d3a8820ac56ec40f3b9e91a0d Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sun, 25 May 2025 15:16:21 +0300 Subject: [PATCH 12/17] Report performance changes up to 1% --- util/bench-compare.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/bench-compare.sh b/util/bench-compare.sh index 0ee61b8..7e829c6 100755 --- a/util/bench-compare.sh +++ b/util/bench-compare.sh @@ -21,10 +21,10 @@ paste -d , "$1" "$2" | printf("%12s encountered an error\n", $1); else if (!$12) printf("%12s encountered an error\n", $2); - else if ($4 / $12 > 1.1) - printf("%12s is %5.1f times faster than %s\n", $2, $4 / $12, $1); - else if ($12 / $4 > 1.1) - printf("%12s is %5.1f times faster than %s\n", $1, $12 / $4, $2); + else if ($4 / $12 > 1.01) + printf("%12s is %5.2f times faster than %s\n", $2, $4 / $12, $1); + else if ($12 / $4 > 1.01) + printf("%12s is %5.2f times faster than %s\n", $1, $12 / $4, $2); else printf("%12s is similarly fast as %s\n", $1, $2); }' From a67be7460466feb4063a6cfe13781c73b731918c Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Sun, 25 May 2025 16:06:22 +0300 Subject: [PATCH 13/17] Specialize single replacement multiple groups Performance improves significantly in the following case: number-fix rust-sed is 1.07 times faster than rust-sed-90d60b8 Other changes <5% are likely to be noise. --- src/uu/sed/src/processor.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/uu/sed/src/processor.rs b/src/uu/sed/src/processor.rs index a9263ce..24253a2 100644 --- a/src/uu/sed/src/processor.rs +++ b/src/uu/sed/src/processor.rs @@ -201,6 +201,7 @@ fn substitute( match (sub.occurrence, sub.replacement.max_group_number) { (1, 0) => { + // Example: s/foo/bar/: find() is enough. let m = regex.find(pattern)?; if let Some(m) = m { text = Some(pattern.as_str()?); @@ -212,7 +213,24 @@ fn substitute( last_end = m.end(); } } + + (1, _) => { + // Example: s/\(.\)\(.\)/\2\1/: captures() is enough. + let caps = regex.captures(pattern)?; + if let Some(caps) = caps { + let m = caps.get(0)?.unwrap(); + text = Some(pattern.as_str()?); + result.push_str(&text.unwrap()[last_end..m.start()]); + + let replacement = sub.replacement.apply_captures(&caps)?; + result.push_str(&replacement); + replaced = true; + last_end = m.end(); + } + } + (_, _) => { + // Example: s/(.)(.)/\2\1/3: captures_iter() is needed. // Iterate over multiple captures of the RE in the pattern. for caps in regex.captures_iter(pattern)? { count += 1; @@ -237,7 +255,8 @@ fn substitute( last_end = m.end(); - // Early exit if only a specific occurrence (likely 1) needed replacing. + // Early exit if only a specific occurrence, + // (likely 1) needed replacing. if count == sub.occurrence { break; } From aa12a127336bb8b92d403ca511a21941419ef1b0 Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Mon, 26 May 2025 20:23:08 +0300 Subject: [PATCH 14/17] Improve fancy_regex detection RE and tests --- src/uu/sed/src/fast_regex.rs | 79 +++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 11 deletions(-) diff --git a/src/uu/sed/src/fast_regex.rs b/src/uu/sed/src/fast_regex.rs index e8e52ed..c023915 100644 --- a/src/uu/sed/src/fast_regex.rs +++ b/src/uu/sed/src/fast_regex.rs @@ -34,20 +34,42 @@ pub enum Regex { impl Regex { /// Construct the most efficient engine possible pub fn new(pattern: &str) -> Result> { + // REs requiring the fancy_regex capabilities rather than the + // regex::bytes engine + // Consider . as one character that requires fancy_regex, + // because it can match more than one byte when matching a + // two or more byte Unicode UTF-8 representation. + // It is an RE . rather than a literal one in the following + // example sitations. + // . First character of the line + // [^\\]. Second character after non \ + // + // \*. A consumed backslash anywhere on the line + // \\. An escaped backslash anywhere on the line + // xx. A non-escaped sequence anywhere on the line + // But the following are literal dots and can be captured by bytes: + // \. escaped at the beginning of the line + // x\. escaped after a non escaped \ anywhere on the line + // + // The following RE captures these situations. static NEEDS_FANCY_RE: Lazy = Lazy::new(|| { regex::Regex::new( r"(?x) # Turn on verbose mode ( # An ASCII-incompatible RE - ( ^ | [^\\] ) # Non-escaped, so BOL or not \ + ( ^ # Non-escaped: i.e. at BOL + | ^[^\\] # or after a BOL non \ + | [^\\] {2} # or after two non \ characters + | \\. # or after a consumed or escaped \ + ) ( # A potentially incompatible match \. # . matches any Unicode character - | \[\^ # Bracketed negative character class - | \(\?i # Case insensitive is done on Unicode + | \[\^ # Bracketed -ve character class + | \(\?i # (Unicode) case insensitive | \\[WwDdSsBbPp] # Unicode classes - | \\[0-9] # Only fancy supports back-references + | \\[0-9] # Back-references need fancy ) ) - | [^\x01-\x7f] # Or any non-ASCII character in the RE + | [^\x01-\x7f] # Any non-ASCII character ", ) .unwrap() @@ -264,16 +286,36 @@ mod tests { ); } + #[test] + fn selects_byte_regex_for_escpaped_dot_bol() { + assert_byte(r"\."); + } + + #[test] + fn selects_byte_regex_for_escpaped_dot_non_bol() { + assert_byte(r"x\."); + } + + #[test] + fn selects_byte_regex_for_escaped_class() { + assert_byte(r"\[^x]"); + } + + #[test] + fn selects_byte_regex_for_escaped_case_insensitive_flag() { + assert_byte(r"\(?i\)"); + } + + #[test] + fn selects_byte_regex_for_escaped_unicode_class() { + assert_byte(r"\\w"); + } + #[test] fn selects_byte_regex_for_simple_ascii() { assert_byte(r"foo"); assert_byte(r"foo|bar"); assert_byte(r"^foo[0-9]+bar$"); - // Escaped Unicode triggers shouldn't trigger RE. - assert_byte(r"\."); - assert_byte(r"\[^x]"); - assert_byte(r"\\(?i)"); - assert_byte(r"\\w"); } #[test] @@ -294,12 +336,27 @@ mod tests { #[test] fn selects_fancy_for_dot() { - assert_fancy(r"."); // Dot matches any Unicode char. + assert_fancy(r"."); + assert_fancy(r"x."); + assert_fancy(r"xx."); + } + + #[test] + fn selects_fancy_for_consumed_backslash() { + assert_fancy(r"\*."); + assert_fancy(r"x\*."); + } + + #[test] + fn selects_fancy_for_escaped_backslash() { + assert_fancy(r"\\."); + assert_fancy(r"x\\."); } #[test] fn selects_fancy_for_inline_flags() { assert_fancy(r"(?i)abc"); // Unicode case-insensitive + assert_fancy(r"x(?i)abc"); // Unicode case-insensitive } #[test] From ea3df65f5c88ef55af8138f77b45453667d9f223 Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Mon, 26 May 2025 20:25:48 +0300 Subject: [PATCH 15/17] Fix Windows warning --- src/uu/sed/src/fast_io.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/uu/sed/src/fast_io.rs b/src/uu/sed/src/fast_io.rs index 2bc678d..273e667 100644 --- a/src/uu/sed/src/fast_io.rs +++ b/src/uu/sed/src/fast_io.rs @@ -14,6 +14,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +#[cfg(unix)] use memchr::memchr; #[cfg(unix)] use memmap2::Mmap; From a069605d7e5a2d17ef1690fb34a88532f151f18b Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Tue, 27 May 2025 20:30:57 +0300 Subject: [PATCH 16/17] Add RE detection and refactor unit tests - Add RE to detect specifications that require an RE engine, rather than a literal string match. - Change RE unit tests to test directly the RE rather than the constructor. - Use arrays for similar tests to avoid repetition. --- src/uu/sed/src/fast_regex.rs | 312 +++++++++++++++++++++-------------- 1 file changed, 185 insertions(+), 127 deletions(-) diff --git a/src/uu/sed/src/fast_regex.rs b/src/uu/sed/src/fast_regex.rs index c023915..ff7760e 100644 --- a/src/uu/sed/src/fast_regex.rs +++ b/src/uu/sed/src/fast_regex.rs @@ -15,6 +15,7 @@ use fancy_regex::{ CaptureMatches as FancyCaptureMatches, Captures as FancyCaptures, Regex as FancyRegex, }; use once_cell::sync::Lazy; +use regex::Regex as RustRegex; use regex::bytes::{ CaptureMatches as ByteCaptureMatches, Captures as ByteCaptures, Regex as ByteRegex, }; @@ -23,6 +24,76 @@ use uucore::error::{UResult, USimpleError}; use crate::fast_io::IOChunk; +/// REs requiring the fancy_regex capabilities rather than the +/// faster regex::bytes engine +// Consider . as one character that requires fancy_regex, +// because it can match more than one byte when matching a +// two or more byte Unicode UTF-8 representation. +// It is an RE . rather than a literal one in the following +// example sitations. +// . First character of the line +// [^\\]. Second character after non \ +// +// \*. A consumed backslash anywhere on the line +// \\. An escaped backslash anywhere on the line +// xx. A non-escaped sequence anywhere on the line +// But the following are literal dots and can be captured by bytes: +// \. escaped at the beginning of the line +// x\. escaped after a non escaped \ anywhere on the line +// +// The following RE captures these situations. +static NEEDS_FANCY_RE: Lazy = Lazy::new(|| { + regex::Regex::new( + r"(?x) # Turn on verbose mode + ( # An ASCII-incompatible RE + ( ^ # Non-escaped: i.e. at BOL + | ^[^\\] # or after a BOL non \ + | [^\\] {2} # or after two non \ characters + | \\. # or after a consumed or escaped \ + ) + ( # A potentially incompatible match + \. # . matches any Unicode character + | \[\^ # Bracketed -ve character class + | \(\?i # (Unicode) case insensitive + | \\[WwDdSsBbPp] # Unicode classes + | \\[0-9] # Back-references need fancy + ) + ) + | [^\x01-\x7f] # Any non-ASCII character + ", + ) + .unwrap() +}); + +/// All characters signifying that the match must be handled by an RE +/// rather than by plain string pattern matching. +// These do not include the ^$ metacharacters, which we can easily handle. +// Plain string fixed-string matching is currently faster than Regex +// matching, because Regex always constructs an automaton and needs +// to handle state transitions, whereas plain string matching can +// use tailored CPU string or vectored instructions. +static NEEDS_RE: Lazy = Lazy::new(|| { + regex::Regex::new( + r"(?x) # Turn on verbose mode + ( ^ # Non-escaped: i.e. at BOL + | ^[^\\] # or after a BOL non \ + | [^\\] {2} # or after two non \ characters + | \\. # or after a consumed or escaped \ + ) + ( # A potentially incompatible match + [.?|+(\[{*] # Any magic RE character + # Some are operators so illegal at + # BOL but they should error there, + # not use them as literals. + | \\[WwDdSsPp] # Unicode classes + | \\[AzBb] # Empty matches + | \\[0-9] # Back-references + ) + ", + ) + .unwrap() +}); + #[derive(Clone, Debug)] /// A regular expression that can be implemented through Byte or Fancy-Regex // This enables a fast bytes path where possible. @@ -34,47 +105,6 @@ pub enum Regex { impl Regex { /// Construct the most efficient engine possible pub fn new(pattern: &str) -> Result> { - // REs requiring the fancy_regex capabilities rather than the - // regex::bytes engine - // Consider . as one character that requires fancy_regex, - // because it can match more than one byte when matching a - // two or more byte Unicode UTF-8 representation. - // It is an RE . rather than a literal one in the following - // example sitations. - // . First character of the line - // [^\\]. Second character after non \ - // - // \*. A consumed backslash anywhere on the line - // \\. An escaped backslash anywhere on the line - // xx. A non-escaped sequence anywhere on the line - // But the following are literal dots and can be captured by bytes: - // \. escaped at the beginning of the line - // x\. escaped after a non escaped \ anywhere on the line - // - // The following RE captures these situations. - static NEEDS_FANCY_RE: Lazy = Lazy::new(|| { - regex::Regex::new( - r"(?x) # Turn on verbose mode - ( # An ASCII-incompatible RE - ( ^ # Non-escaped: i.e. at BOL - | ^[^\\] # or after a BOL non \ - | [^\\] {2} # or after two non \ characters - | \\. # or after a consumed or escaped \ - ) - ( # A potentially incompatible match - \. # . matches any Unicode character - | \[\^ # Bracketed -ve character class - | \(\?i # (Unicode) case insensitive - | \\[WwDdSsBbPp] # Unicode classes - | \\[0-9] # Back-references need fancy - ) - ) - | [^\x01-\x7f] # Any non-ASCII character - ", - ) - .unwrap() - }); - if NEEDS_FANCY_RE.is_match(pattern) { Ok(Self::Fancy(FancyRegex::new(pattern)?)) } else { @@ -272,102 +302,130 @@ impl<'t> Captures<'t> { mod tests { use super::*; - fn assert_byte(pattern: &str) { - let re = Regex::new(pattern).unwrap(); - assert!(re.is_byte(), "Pattern {:?} should use Byte engine", pattern); - } + // FANCY_RE + #[test] + fn test_needs_fancy_re_matches() { + let should_match = [ + // Unicode classes BOL + r"\p{L}+", // Unicode letter class + r"\W", // \W is Unicode-aware. + r"\S+", // \S is Unicode-aware. + r"\d", // \d includes all Unicode digits. + // Unicode classes non-BOL + r"x\p{L}+", // Unicode letter class + r"x\W", // \W is Unicode-aware. + r"x\S+", // \S is Unicode-aware. + r"x\d", // \d includes all Unicode digits. + // . + r".", + r"x.", + r"xx.", + // Consumed \ + r"\*.", + r"x\*.", + // Escaped \ + r"\\.", + r"x\\.", + // Inline flags + r"(?i)abc", // Unicode case-insensitive + r"x(?i)abc", // Unicode case-insensitive + r"(\w+):\1", // back-reference \1 + // Non-ASCII literals + "naïve", // Contains literal non-ASCII. + "café", // Contains literal non-ASCII. + ]; - fn assert_fancy(pattern: &str) { - let re = Regex::new(pattern).unwrap(); - assert!( - !re.is_byte(), - "Pattern {:?} should use Fancy engine", - pattern - ); + for pat in &should_match { + assert!( + NEEDS_FANCY_RE.is_match(pat), + "Expected NEEDS_FANCY_RE to match: {:?}", + pat + ); + } } #[test] - fn selects_byte_regex_for_escpaped_dot_bol() { - assert_byte(r"\."); + fn test_needs_fancy_re_does_not_match() { + let should_not_match = [ + r"\.", // Escaped . at BOL + r"x\.", // Escaped . at non BOL + r"\[^x]", // Escaped character class + r"\(?i\)", // Escaped case insesitive flag + r"\\w", // Escaped Unicode class + // Simple ASCII + r"foo", + r"foo|bar", + r"^foo[0-9]+bar$", + ]; + + for pat in &should_not_match { + assert!( + !NEEDS_FANCY_RE.is_match(pat), + "Expected NEEDS_FANCY_RE to NOT match: {:?}", + pat + ); + } + } + + // NEEDS_RE + #[test] + fn test_needs_re_matches() { + let should_match = [ + r".", // Single regex wildcard + r"a+b", // Regex + + r"foo|bar", // Regex alternation + r"abc?", // Regex optional + r"a*b", // Regex star + r"[abc]", // Character class + r"(abc)", // Group + r"{1,2}", // Repetition + r"\d", // Class shorthand + r"\S", // Class shorthand + r"\1", // Backreference + r"a\Pb", // Unicode property + ]; + + for pat in &should_match { + assert!( + NEEDS_RE.is_match(pat), + "Expected NEEDS_RE to match: {:?}", + pat + ); + } } #[test] - fn selects_byte_regex_for_escpaped_dot_non_bol() { - assert_byte(r"x\."); + fn test_needs_re_does_not_match() { + let should_not_match = [ + r"abc", + r"a\.b", // Escaped dot + r"hello world", + r"^abc$", // Anchors alone + r"file\.", // Escaped dot + r"literal123", + r"\\", // Escaped backslash + ]; + + for pat in &should_not_match { + assert!( + !NEEDS_RE.is_match(pat), + "Expected NEEDS_RE to NOT match: {:?}", + pat + ); + } + } + + // Regex::new + #[test] + fn assert_byte_selection() { + let re = Regex::new(r"x\.").unwrap(); + assert!(re.is_byte()); } #[test] - fn selects_byte_regex_for_escaped_class() { - assert_byte(r"\[^x]"); - } - - #[test] - fn selects_byte_regex_for_escaped_case_insensitive_flag() { - assert_byte(r"\(?i\)"); - } - - #[test] - fn selects_byte_regex_for_escaped_unicode_class() { - assert_byte(r"\\w"); - } - - #[test] - fn selects_byte_regex_for_simple_ascii() { - assert_byte(r"foo"); - assert_byte(r"foo|bar"); - assert_byte(r"^foo[0-9]+bar$"); - } - - #[test] - fn selects_fancy_for_unicode_class_bol() { - assert_fancy(r"\p{L}+"); // Unicode letter class - assert_fancy(r"\W"); // \W is Unicode-aware. - assert_fancy(r"\S+"); // \S is Unicode-aware. - assert_fancy(r"\d"); // \d includes all Unicode digits. - } - - #[test] - fn selects_fancy_for_unicode_class_non_bol() { - assert_fancy(r"x\p{L}+"); // Unicode letter class - assert_fancy(r"x\W"); // \W is Unicode-aware. - assert_fancy(r"x\S+"); // \S is Unicode-aware. - assert_fancy(r"x\d"); // \d includes all Unicode digits. - } - - #[test] - fn selects_fancy_for_dot() { - assert_fancy(r"."); - assert_fancy(r"x."); - assert_fancy(r"xx."); - } - - #[test] - fn selects_fancy_for_consumed_backslash() { - assert_fancy(r"\*."); - assert_fancy(r"x\*."); - } - - #[test] - fn selects_fancy_for_escaped_backslash() { - assert_fancy(r"\\."); - assert_fancy(r"x\\."); - } - - #[test] - fn selects_fancy_for_inline_flags() { - assert_fancy(r"(?i)abc"); // Unicode case-insensitive - assert_fancy(r"x(?i)abc"); // Unicode case-insensitive - } - - #[test] - fn selects_fancy_for_backrefs() { - assert_fancy(r"(\w+):\1"); // back-reference \1 - } - - #[test] - fn selects_fancy_for_non_ascii_literals() { - assert_fancy("naïve"); // Contains literal non-ASCII. - assert_fancy("café"); // Contains literal non-ASCII. + fn assert_fancy() { + let re = Regex::new(r"\d").unwrap(); + assert!(!re.is_byte()); } #[test] From 5af9cf66b317061e67b424be67e6c55fd7ff372f Mon Sep 17 00:00:00 2001 From: Diomidis Spinellis Date: Wed, 28 May 2025 14:35:55 +0300 Subject: [PATCH 17/17] Implement an efficient literal string matcher Regex uses an automaton even for matching literal strings. Moving through its state transitions is suboptimal. This commit introduces literal string matching with the memchr::memmem matcher, which may use SIMD and other specialized features to speed up the search as well as other algorithms for special sizes. This boosts substantially the performance of several benchmark cases. access-log-no-subst rust-86 is 2.84 times faster than rust-sed-a6 access-log-subst rust-86 is 1.87 times faster than rust-sed-a6 access-log-no-del rust-86 is 3.00 times faster than rust-sed-a6 access-log-all-del rust-86 is 2.80 times faster than rust-sed-a6 remove-cr rust-86 is 4.35 times faster than rust-sed-a6 genome-subst rust-86 is 3.65 times faster than rust-sed-a6 It makes makes the performance of Rust better than the GNU and FreeBSD implementations for most benchmark cases. no-op-short rust is 3.04 times faster than gnu access-log-no-op rust is 1.91 times faster than gnu access-log-no-subst rust is 1.59 times faster than gnu access-log-subst rust is 1.32 times faster than gnu access-log-no-del rust is 1.54 times faster than gnu access-log-all-del rust is 1.99 times faster than gnu access-log-translit rust is 12.67 times faster than gnu access-log-complex-sub gnu is 4.24 times faster than rust remove-cr rust is 1.68 times faster than gnu genome-subst rust is 1.15 times faster than gnu number-fix rust is 1.06 times faster than gnu long-script gnu is 2.19 times faster than rust hanoi rust is 1.96 times faster than gnu factorial rust is 1.25 times faster than gnu no-op-short rust is 3.50 times faster than fbsd access-log-no-op rust is 2.45 times faster than fbsd access-log-no-subst rust is 1.38 times faster than fbsd access-log-subst rust is 4.02 times faster than fbsd access-log-no-del rust is 1.35 times faster than fbsd access-log-all-del rust is 4.74 times faster than fbsd access-log-translit fbsd is 1.14 times faster than rust access-log-complex-sub rust is 2.00 times faster than fbsd remove-cr rust is 2.48 times faster than fbsd genome-subst rust is 2.21 times faster than fbsd number-fix fbsd is 1.04 times faster than rust long-script fbsd is 15.46 times faster than rust hanoi rust is 9.33 times faster than fbsd factorial rust is 115.72 times faster than fbsd --- src/uu/sed/src/fast_regex.rs | 341 +++++++++++++++++++++++++++++++++-- 1 file changed, 323 insertions(+), 18 deletions(-) diff --git a/src/uu/sed/src/fast_regex.rs b/src/uu/sed/src/fast_regex.rs index ff7760e..b45d36c 100644 --- a/src/uu/sed/src/fast_regex.rs +++ b/src/uu/sed/src/fast_regex.rs @@ -14,6 +14,7 @@ use fancy_regex::{ CaptureMatches as FancyCaptureMatches, Captures as FancyCaptures, Regex as FancyRegex, }; +use memchr::memmem; use once_cell::sync::Lazy; use regex::Regex as RustRegex; use regex::bytes::{ @@ -95,31 +96,166 @@ static NEEDS_RE: Lazy = Lazy::new(|| { }); #[derive(Clone, Debug)] -/// A regular expression that can be implemented through Byte or Fancy-Regex -// This enables a fast bytes path where possible. -pub enum Regex { - Byte(ByteRegex), - Fancy(FancyRegex), +/// Types of literal string anchored matches +enum AnchoredMatch { + Begin, // ^... + End, // ...$ + Both, // ^...$ + Free, // ... } -impl Regex { - /// Construct the most efficient engine possible - pub fn new(pattern: &str) -> Result> { - if NEEDS_FANCY_RE.is_match(pattern) { - Ok(Self::Fancy(FancyRegex::new(pattern)?)) +#[derive(Clone, Debug)] +/// A fast Regex-like matcher for literal strings using memchr:memmem +pub struct LiteralMatcher { + needle: Vec, // Bytes without any anchors + match_type: AnchoredMatch, // Type of anchoring specified +} + +impl LiteralMatcher { + /// Construct a new matcher based on a needle possible with anchors. + pub fn new(needle: &str) -> Self { + let needle_bytes = needle.as_bytes(); + if needle_bytes[0] == b'^' && needle_bytes[needle_bytes.len() - 1] == b'$' { + LiteralMatcher { + match_type: AnchoredMatch::Both, + needle: needle_bytes[1..needle_bytes.len() - 1].to_vec(), + } + } else if needle_bytes[0] == b'^' { + LiteralMatcher { + match_type: AnchoredMatch::Begin, + needle: needle_bytes[1..needle_bytes.len()].to_vec(), + } + } else if needle_bytes[needle_bytes.len() - 1] == b'$' { + LiteralMatcher { + match_type: AnchoredMatch::End, + needle: needle_bytes[0..needle_bytes.len() - 1].to_vec(), + } } else { - Ok(Self::Byte(ByteRegex::new(pattern)?)) + LiteralMatcher { + match_type: AnchoredMatch::Free, + needle: needle_bytes.to_vec(), + } } } - /// Return true if this is a byte-based regex. - pub fn is_byte(&self) -> bool { - matches!(self, Regex::Byte(_)) + /// Returns the start index of a match, if any + fn anchored_find(&self, haystack: &[u8]) -> Option { + let nlen = self.needle.len(); + let hlen = haystack.len(); + + match self.match_type { + AnchoredMatch::Both => { + if hlen == nlen && haystack == self.needle.as_slice() { + Some(0) + } else { + None + } + } + AnchoredMatch::Begin => { + if hlen >= nlen && &haystack[..nlen] == self.needle.as_slice() { + Some(0) + } else { + None + } + } + AnchoredMatch::End => { + if hlen >= nlen && &haystack[hlen - nlen..] == self.needle.as_slice() { + Some(hlen - nlen) + } else { + None + } + } + AnchoredMatch::Free => memmem::find(haystack, &self.needle), + } + } + + /// Return true if the needle occurs in the haystack. + pub fn is_match(&self, haystack: &[u8]) -> bool { + self.anchored_find(haystack).is_some() + } + + /// Return the position and contents of the matched needle. + pub fn find<'t>(&self, haystack: &'t [u8]) -> Option<(usize, usize, &'t str)> { + self.anchored_find(haystack).and_then(|start| { + let end = start + self.needle.len(); + std::str::from_utf8(&haystack[start..end]) + .ok() + .map(|s| (start, end, s)) + }) + } + + /// Return all positions and contents of the matched needle. + pub fn iter<'t>( + &'t self, + haystack: &'t [u8], + ) -> Box + 't> { + let needle = &self.needle; + let nlen = needle.len(); + + match self.match_type { + AnchoredMatch::Both | AnchoredMatch::Begin | AnchoredMatch::End => { + // At most one match; yield it if present + Box::new(self.find(haystack).into_iter()) + } + AnchoredMatch::Free => { + // Multiple potential matches + Box::new( + memmem::find_iter(haystack, needle).filter_map(move |start| { + let end = start + nlen; + std::str::from_utf8(&haystack[start..end]) + .ok() + .map(|s| (start, end, s)) + }), + ) + } + } + } +} + +/// Return the passed pattern without any backslash escapes. +pub fn remove_escapes(pattern: &str) -> String { + let mut chars = pattern.chars().peekable(); + let mut result = String::with_capacity(pattern.len()); + + while let Some(c) = chars.next() { + if c == '\\' { + // Look ahead and consume the next character if present + if let Some(&next) = chars.peek() { + result.push(next); + chars.next(); // consume the peeked char + } + } else { + result.push(c); + } + } + + result +} + +#[derive(Clone, Debug)] +/// A regular expression that can be implemented in diverse efficient ways +pub enum Regex { + Literal(LiteralMatcher), // Fastest: literal bytes + Byte(ByteRegex), // Slower: byte-based RE + Fancy(FancyRegex), // Slowest: RE supporting UTF-8 and back-references +} + +impl Regex { + /// Construct the most efficient RE-like matching engine possible. + pub fn new(pattern: &str) -> Result> { + if NEEDS_FANCY_RE.is_match(pattern) { + Ok(Self::Fancy(FancyRegex::new(pattern)?)) + } else if NEEDS_RE.is_match(pattern) { + Ok(Self::Byte(ByteRegex::new(pattern)?)) + } else { + Ok(Self::Literal(LiteralMatcher::new(&remove_escapes(pattern)))) + } } /// Check if the regex matches the content of the IOChunk. pub fn is_match(&self, chunk: &mut IOChunk) -> UResult { match self { + Regex::Literal(m) => Ok(m.is_match(chunk.as_bytes())), Regex::Byte(re) => Ok(re.is_match(chunk.as_bytes())), Regex::Fancy(re) => { let text = chunk.as_str()?; @@ -132,7 +268,15 @@ impl Regex { /// Return an iterator over capture groups. pub fn captures_iter<'t>(&'t self, chunk: &'t IOChunk) -> UResult> { match self { + Regex::Literal(m) => { + let haystack = chunk.as_bytes(); + Ok(CaptureMatches::Literal(Box::new(m.iter(haystack).map( + |(start, end, text)| Ok(Captures::Literal(Match { start, end, text })), + )))) + } + Regex::Byte(re) => Ok(CaptureMatches::Byte(re.captures_iter(chunk.as_bytes()))), + Regex::Fancy(re) => { let text = chunk.as_str()?; Ok(CaptureMatches::Fancy(re.captures_iter(text))) @@ -143,6 +287,7 @@ impl Regex { /// Return the number of capture groups, including group 0. pub fn captures_len(&self) -> usize { match self { + Regex::Literal(_) => 1, // Only group 0 Regex::Byte(re) => re.captures_len(), Regex::Fancy(re) => re.captures_len(), } @@ -151,6 +296,16 @@ impl Regex { /// Return the elements of the first capture. pub fn captures<'t>(&self, chunk: &'t IOChunk) -> UResult>> { match self { + Regex::Literal(m) => { + let haystack = chunk.as_bytes(); + match m.find(haystack) { + Some((start, end, text)) => { + Ok(Some(Captures::Literal(Match { start, end, text }))) + } + None => Ok(None), + } + } + Regex::Byte(re) => { let bytes = chunk.as_bytes(); Ok(re.captures(bytes).map(Captures::Byte)) @@ -170,6 +325,14 @@ impl Regex { /// Return a non-capturing result for a single match. pub fn find<'t>(&self, chunk: &'t IOChunk) -> UResult>> { match self { + Regex::Literal(m) => { + let haystack = chunk.as_bytes(); + match m.find(haystack) { + Some((start, end, text)) => Ok(Some(Match { start, end, text })), + None => Ok(None), + } + } + Regex::Byte(re) => { let haystack = chunk.as_bytes(); if let Some(m) = re.find(haystack) { @@ -185,6 +348,7 @@ impl Regex { Ok(None) } } + Regex::Fancy(re) => { let text = chunk.as_str()?; match re.find(text) { @@ -203,6 +367,7 @@ impl Regex { /// Unified enum for holding either byte or fancy capture iterators. pub enum CaptureMatches<'t> { + Literal(Box>> + 't>), Byte(ByteCaptureMatches<'t, 't>), Fancy(FancyCaptureMatches<'t, 't>), } @@ -212,6 +377,7 @@ impl<'t> Iterator for CaptureMatches<'t> { fn next(&mut self) -> Option { match self { + CaptureMatches::Literal(iter) => iter.next(), CaptureMatches::Byte(iter) => iter.next().map(|caps| Ok(Captures::Byte(caps))), CaptureMatches::Fancy(iter) => match iter.next() { Some(Ok(caps)) => Some(Ok(Captures::Fancy(caps))), @@ -225,7 +391,7 @@ impl<'t> Iterator for CaptureMatches<'t> { } } -#[derive(Debug)] +#[derive(Clone, Debug)] /// Result type for RE capture get(n) pub struct Match<'t> { start: usize, // Match start @@ -250,6 +416,7 @@ impl<'t> Match<'t> { /// Provide interface compatible with Regex::Captures. pub enum Captures<'t> { + Literal(Match<'t>), // only group 0 Byte(ByteCaptures<'t>), Fancy(FancyCaptures<'t>), } @@ -260,6 +427,7 @@ impl<'t> Captures<'t> { /// Returns Err if UTF-8 conversion fails (in Byte variant). pub fn get(&self, i: usize) -> UResult>> { match self { + Captures::Literal(m) => Ok(if i == 0 { Some(m.clone()) } else { None }), Captures::Byte(caps) => match caps.get(i) { Some(m) => Ok(Some(Match { start: m.start(), @@ -283,6 +451,7 @@ impl<'t> Captures<'t> { /// Return the number of capture groups (including group 0). pub fn len(&self) -> usize { match self { + Captures::Literal(_) => 1, Captures::Byte(caps) => caps.len(), Captures::Fancy(caps) => caps.len(), } @@ -292,6 +461,7 @@ impl<'t> Captures<'t> { // Unused, but provided for completeness. pub fn is_empty(&self) -> bool { match self { + Captures::Literal(_) => false, // A literal match always has group 0 Captures::Byte(caps) => caps.len() == 0, Captures::Fancy(caps) => caps.len() == 0, } @@ -418,14 +588,20 @@ mod tests { // Regex::new #[test] fn assert_byte_selection() { - let re = Regex::new(r"x\.").unwrap(); - assert!(re.is_byte()); + let re = Regex::new(r"x*").unwrap(); + assert!(matches!(re, Regex::Byte(_))); } #[test] fn assert_fancy() { let re = Regex::new(r"\d").unwrap(); - assert!(!re.is_byte()); + assert!(matches!(re, Regex::Fancy(_))); + } + + #[test] + fn assert_literal() { + let re = Regex::new(r"x\.").unwrap(); + assert!(matches!(re, Regex::Literal(_))); } #[test] @@ -437,4 +613,133 @@ mod tests { err ); } + + // remove_escapes + #[test] + fn test_remove_escapes() { + use super::remove_escapes; + + assert_eq!(remove_escapes("abc"), "abc"); + assert_eq!(remove_escapes(r"a\.c"), "a.c"); + assert_eq!(remove_escapes(r"\\d"), r"\d"); + assert_eq!(remove_escapes(r"\.\*\+\?"), ".*+?"); + assert_eq!(remove_escapes(r"escaped\\backslash"), r"escaped\backslash"); + assert_eq!(remove_escapes(r"trailing\\"), r"trailing\"); + } + + // LiteralMatcher + #[test] + fn test_literal_matcher_basic_match() { + let matcher = LiteralMatcher::new("needle"); + assert!(matcher.is_match(b"this is a needle in a haystack")); + assert!(!matcher.is_match(b"no match here")); + } + + #[test] + fn test_literal_matcher_anchor_start_match() { + let matcher = LiteralMatcher::new("^needle"); + assert!(matcher.is_match(b"needle in a haystack")); + assert!(!matcher.is_match(b"no needle match here")); + assert!(!matcher.is_match(b"no")); + } + + #[test] + fn test_literal_matcher_anchor_end_match() { + let matcher = LiteralMatcher::new("needle$"); + assert!(matcher.is_match(b"In a haystack there's a needle")); + assert!(!matcher.is_match(b"no needle match here")); + assert!(!matcher.is_match(b"no")); + } + + #[test] + fn test_literal_matcher_anchor_begin_end_match() { + let matcher = LiteralMatcher::new("^needle$"); + assert!(matcher.is_match(b"needle")); + assert!(!matcher.is_match(b"no needle match")); + assert!(!matcher.is_match(b"needle no match")); + assert!(!matcher.is_match(b"no match needle")); + assert!(!matcher.is_match(b"nada")); + } + + #[test] + fn test_literal_matcher_utf8_match() { + let matcher = LiteralMatcher::new("✓"); // U+2713 CHECK MARK (3 bytes) + let haystack = "contains ✓ unicode".as_bytes(); + assert!(matcher.is_match(haystack)); + let found = matcher.find(haystack).unwrap(); + assert_eq!(found.2, "✓"); + } + + #[test] + fn test_literal_matcher_find_location() { + let matcher = LiteralMatcher::new("abc"); + let haystack = b"___abc___"; + let result = matcher.find(haystack); + assert!(result.is_some()); + let (start, end, text) = result.unwrap(); + assert_eq!((start, end), (3, 6)); + assert_eq!(text, "abc"); + } + + #[test] + fn test_literal_matcher_find_location_end() { + let matcher = LiteralMatcher::new("abc$"); + let haystack = b"012abc"; + let result = matcher.find(haystack); + assert!(result.is_some()); + let (start, end, text) = result.unwrap(); + assert_eq!((start, end), (3, 6)); + assert_eq!(text, "abc"); + } + + #[test] + fn test_literal_matcher_iter_multiple() { + let matcher = LiteralMatcher::new("test"); + let haystack = b"this test is a test of test matching"; + let matches: Vec<_> = matcher.iter(haystack).collect(); + assert_eq!(matches.len(), 3); + + let strings: Vec<_> = matches.iter().map(|(_, _, s)| *s).collect(); + assert_eq!(strings, ["test", "test", "test"]); + } + + #[test] + fn test_literal_matcher_iter_begin() { + let matcher = LiteralMatcher::new("^test"); + let haystack = b"test is a test of test matching"; + let matches: Vec<_> = matcher.iter(haystack).collect(); + assert_eq!(matches.len(), 1); + + let strings: Vec<_> = matches.iter().map(|(_, _, s)| *s).collect(); + assert_eq!(strings, ["test"]); + } + + #[test] + fn test_literal_matcher_iter_end() { + let matcher = LiteralMatcher::new("test$"); + let haystack = b"this test is a test of test"; + let matches: Vec<_> = matcher.iter(haystack).collect(); + assert_eq!(matches.len(), 1); + + let strings: Vec<_> = matches.iter().map(|(_, _, s)| *s).collect(); + assert_eq!(strings, ["test"]); + } + + #[test] + fn test_literal_matcher_no_match() { + let matcher = LiteralMatcher::new("missing"); + let haystack = b"nothing to see here"; + assert!(!matcher.is_match(haystack)); + assert!(matcher.find(haystack).is_none()); + assert_eq!(matcher.iter(haystack).count(), 0); + } + + #[test] + fn test_literal_matcher_anchored_no_match() { + let matcher = LiteralMatcher::new("^see$"); + let haystack = b"nothing to see here"; + assert!(!matcher.is_match(haystack)); + assert!(matcher.find(haystack).is_none()); + assert_eq!(matcher.iter(haystack).count(), 0); + } }