diff --git a/Cargo.lock b/Cargo.lock index e87e1f3..10be09f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -703,7 +703,9 @@ dependencies = [ "ctor", "fancy-regex", "libc", + "memchr", "memmap2", + "once_cell", "phf", "phf_codegen", "pretty_assertions", @@ -896,7 +898,9 @@ version = "0.0.1" dependencies = [ "clap", "fancy-regex", + "memchr", "memmap2", + "once_cell", "regex", "tempfile", "uucore", diff --git a/Cargo.toml b/Cargo.toml index 871f72c..ca18467 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,9 @@ clap_complete = "4.5" clap_mangen = "0.2" fancy-regex = "0.14.0" libc = "0.2.153" +memchr = "2.7.4" memmap2 = "0.9" +once_cell = "1.21.3" phf = "0.11.2" phf_codegen = "0.11.2" rand = { version = "0.9", features = ["small_rng"] } @@ -56,7 +58,9 @@ clap_complete = { workspace = true } clap_mangen = { workspace = true } ctor = "0.4.1" fancy-regex = { workspace = true } +memchr = { workspace = true } memmap2.workspace = true +once_cell = { workspace = true } phf = { workspace = true } sed = { optional = true, version = "0.0.1", package = "uu_sed", path = "src/uu/sed" } sysinfo = { workspace = true } diff --git a/README.md b/README.md index c419bab..495adcf 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ cargo run --release * In addition to `\n`, other escape sequences (octal, hex, C) are supported in the strings of the `y` command. Under POSIX these yield undefined behavior. +* The substitution command replacement group `\0` is a synonym for &. ### Supported BSD and GNU extensions * The second address in a range can be specified as a relative address with +N. diff --git a/src/uu/sed/Cargo.toml b/src/uu/sed/Cargo.toml index 6d94616..acf2718 100644 --- a/src/uu/sed/Cargo.toml +++ b/src/uu/sed/Cargo.toml @@ -15,9 +15,11 @@ categories = ["command-line-utilities"] [dependencies] clap = { workspace = true } fancy-regex = { workspace = true } +memchr = { workspace = true } regex = { workspace = true } tempfile = { workspace = true } memmap2 = { workspace = true } +once_cell = { workspace = true } uucore = { workspace = true } [lib] diff --git a/src/uu/sed/src/command.rs b/src/uu/sed/src/command.rs index 24a8038..28e30e0 100644 --- a/src/uu/sed/src/command.rs +++ b/src/uu/sed/src/command.rs @@ -11,15 +11,15 @@ // TODO: remove when compile is implemented #![allow(dead_code)] +use crate::fast_regex::{Captures, Match, Regex}; use crate::named_writer::NamedWriter; -use fancy_regex::{Captures, Regex}; use std::borrow::Cow; use std::cell::RefCell; use std::collections::HashMap; use std::path::PathBuf; // For file descriptors and equivalent use std::rc::Rc; -use uucore::error::UResult; +use uucore::error::{UResult, USimpleError}; #[derive(Debug, Default, Clone)] /// Compilation and processing options provided mostly through the @@ -124,38 +124,65 @@ pub enum ReplacementPart { /// All specified replacements for an RE pub struct ReplacementTemplate { pub parts: Vec, + pub max_group_number: usize, // Highest used group number (e.g. 8 for \8) } impl Default for ReplacementTemplate { /// Create an empty template. fn default() -> Self { - ReplacementTemplate { parts: Vec::new() } + ReplacementTemplate::new(Vec::new()) } } impl ReplacementTemplate { + /// Construct from the parts + pub fn new(parts: Vec) -> Self { + let max_group_number = parts + .iter() + .filter_map(|part| match part { + ReplacementPart::Group(n) => Some(*n), + _ => None, + }) + .max() + .unwrap_or(0); + + Self { + parts, + max_group_number: max_group_number.try_into().unwrap(), + } + } + /// Apply the template to the given RE captures. /// Example: /// let result = regex.replace_all(input, |caps: &Captures| { - /// template.apply(caps) }); + /// template.apply_captures(caps) }); /// Returns an error if a backreference in the template was not matched by the RE. - pub fn apply(&self, caps: &Captures) -> UResult { + pub fn apply_captures(&self, caps: &Captures) -> UResult { let mut result = String::new(); + // Invalid group numbers may end here through (unkown at compile time) + // reused REs. + if self.max_group_number > caps.len() - 1 { + return Err(USimpleError::new( + 2, + format!( + "invalid reference \\{} on `s' command's RHS", + self.max_group_number + ), + )); + } + for part in &self.parts { match part { ReplacementPart::Literal(s) => result.push_str(s), ReplacementPart::WholeMatch => { - result.push_str(caps.get(0).map_or("", |m| m.as_str())); + result.push_str(caps.get(0)?.map(|m| m.as_str()).unwrap_or("")); } ReplacementPart::Group(n) => { - // Compilation guarantees we only get valid group numbers - result.push_str( - caps.get((*n).try_into().unwrap()) - .map_or("", |m| m.as_str()), - ); + let i: usize = (*n).try_into().unwrap(); + result.push_str(caps.get(i)?.map(|m| m.as_str()).unwrap_or("")); } } } @@ -163,19 +190,22 @@ impl ReplacementTemplate { Ok(result) } - /// Returns the highest capture group number referenced in this template. - pub fn max_group_number(&self) -> u32 { - self.parts - .iter() - .filter_map(|part| { - if let ReplacementPart::Group(n) = part { - Some(*n) - } else { - None + /// Apply the template to the given RE single match. + pub fn apply_match(&self, m: &Match) -> String { + let mut result = String::new(); + + for part in &self.parts { + match part { + ReplacementPart::Literal(s) => result.push_str(s), + + ReplacementPart::WholeMatch => result.push_str(m.as_str()), + + ReplacementPart::Group(_) => { + panic!("unexpected Regex group replacement") } - }) - .max() - .unwrap_or(0) + } + } + result } } @@ -322,12 +352,13 @@ pub struct InputAction { #[cfg(test)] mod tests { use super::*; + use crate::fast_io::IOChunk; // Return the captures for the RE applied to the specified string - fn caps_for<'a>(re: &str, input: &'a str) -> Captures<'a> { + fn caps_for<'a>(re: &str, chunk: &'a mut IOChunk) -> Captures<'a> { Regex::new(re) .unwrap() - .captures(input) + .captures(chunk) .unwrap() .expect("captures") } @@ -336,96 +367,108 @@ mod tests { // s/foo// fn test_empty_template() { let template = ReplacementTemplate::default(); - let caps = caps_for("foo", "foo"); + let input = &mut IOChunk::from_str("foo"); + let caps = caps_for("foo", input); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, ""); } #[test] // s/abc/hello/ fn test_literal_only() { - let template = ReplacementTemplate { - parts: vec![ReplacementPart::Literal("hello".into())], - }; - let caps = caps_for("abc", "abc"); + let template = ReplacementTemplate::new(vec![ReplacementPart::Literal("hello".into())]); + let input = &mut IOChunk::from_str("abc"); + let caps = caps_for("abc", input); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "hello"); } #[test] // s/foo\d+/got: &/ fn test_whole_match() { - let template = ReplacementTemplate { - parts: vec![ - ReplacementPart::Literal("got: ".into()), - ReplacementPart::WholeMatch, - ], - }; - let caps = caps_for(r"foo\d+", "foo42"); + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("got: ".into()), + ReplacementPart::WholeMatch, + ]); + let input = &mut IOChunk::from_str("foo42"); + let caps = caps_for(r"foo\d+", input); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "got: foo42"); } #[test] // s/foo(\d+)/number: \1/ fn test_backreference() { - let template = ReplacementTemplate { - parts: vec![ - ReplacementPart::Literal("number: ".into()), - ReplacementPart::Group(1), - ], - }; - let caps = caps_for(r"foo(\d+)", "foo42"); + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("number: ".into()), + ReplacementPart::Group(1), + ]); + let input = &mut IOChunk::from_str("foo42"); + let caps = caps_for(r"foo(\d+)", input); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "number: 42"); } #[test] // s/(\w+):(\d+)/key: \1, value: \2/ fn test_multiple_parts() { - let template = ReplacementTemplate { - parts: vec![ - ReplacementPart::Literal("key: ".into()), - ReplacementPart::Group(1), - ReplacementPart::Literal(", value: ".into()), - ReplacementPart::Group(2), - ], - }; - let caps = caps_for(r"(\w+):(\d+)", "x:123"); + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("key: ".into()), + ReplacementPart::Group(1), + ReplacementPart::Literal(", value: ".into()), + ReplacementPart::Group(2), + ]); + let input = &mut IOChunk::from_str("x:123"); + let caps = caps_for(r"(\w+):(\d+)", input); - let result = template.apply(&caps).unwrap(); + let result = template.apply_captures(&caps).unwrap(); assert_eq!(result, "key: x, value: 123"); } + #[test] + // s/(\w+):(\d+)/key: \1, value: \3/ + fn test_invalid_group() { + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("key: ".into()), + ReplacementPart::Group(1), + ReplacementPart::Literal(", value: ".into()), + ReplacementPart::Group(3), + ]); + let input = &mut IOChunk::from_str("x:123"); + let caps = caps_for(r"(\w+):(\d+)", input); + + let result = template.apply_captures(&caps); + assert!(result.is_err()); + + let msg = result.unwrap_err().to_string(); + assert!(msg.contains("invalid reference \\3")); + } + // max_group_number #[test] fn test_max_group_number_with_groups() { - let template = ReplacementTemplate { - parts: vec![ - ReplacementPart::Literal("a".into()), - ReplacementPart::Group(2), - ReplacementPart::WholeMatch, - ReplacementPart::Group(5), - ReplacementPart::Literal("z".into()), - ], - }; - assert_eq!(template.max_group_number(), 5); + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("a".into()), + ReplacementPart::Group(2), + ReplacementPart::WholeMatch, + ReplacementPart::Group(5), + ReplacementPart::Literal("z".into()), + ]); + assert_eq!(template.max_group_number, 5); } #[test] fn test_max_group_number_without_groups() { - let template = ReplacementTemplate { - parts: vec![ - ReplacementPart::Literal("no".into()), - ReplacementPart::WholeMatch, - ReplacementPart::Literal("groups".into()), - ], - }; - assert_eq!(template.max_group_number(), 0); + let template = ReplacementTemplate::new(vec![ + ReplacementPart::Literal("no".into()), + ReplacementPart::WholeMatch, + ReplacementPart::Literal("groups".into()), + ]); + assert_eq!(template.max_group_number, 0); } // Transliteration diff --git a/src/uu/sed/src/compiler.rs b/src/uu/sed/src/compiler.rs index cdfc135..d3a7c28 100644 --- a/src/uu/sed/src/compiler.rs +++ b/src/uu/sed/src/compiler.rs @@ -15,10 +15,10 @@ use crate::command::{ use crate::delimited_parser::{ compilation_error, parse_char_escape, parse_regex, parse_transliteration, }; +use crate::fast_regex::Regex; use crate::named_writer::NamedWriter; use crate::script_char_provider::ScriptCharProvider; use crate::script_line_provider::ScriptLineProvider; -use fancy_regex::Regex; use std::borrow::Cow; use std::cell::RefCell; use std::collections::HashMap; @@ -706,14 +706,18 @@ pub fn compile_replacement( } match line.current() { - // \1 - \9 - c @ '1'..='9' => { + // \0 - \9 + c @ '0'..='9' => { let ref_num = c.to_digit(10).unwrap(); if !literal.is_empty() { parts.push(ReplacementPart::Literal(std::mem::take(&mut literal))); } - parts.push(ReplacementPart::Group(ref_num)); + if ref_num == 0 { + parts.push(ReplacementPart::WholeMatch); + } else { + parts.push(ReplacementPart::Group(ref_num)); + } line.advance(); } @@ -762,7 +766,7 @@ pub fn compile_replacement( if !literal.is_empty() { parts.push(ReplacementPart::Literal(literal)); } - return Ok(ReplacementTemplate { parts }); + return Ok(ReplacementTemplate::new(parts)); } c => { @@ -816,11 +820,21 @@ fn compile_subst_command( ); } - if pattern.is_empty() { - subst.regex = None; // Will be reuse saved RE at runtime. - } else { - // Compile regex with now known ignore_case flag. - subst.regex = compile_regex(lines, line, &pattern, context, subst.ignore_case)?; + // Compile regex with now known ignore_case flag. + subst.regex = compile_regex(lines, line, &pattern, context, subst.ignore_case)?; + + // Catch invalid group references at compile time, if possible. + if let Some(regex) = &subst.regex { + if subst.replacement.max_group_number > regex.captures_len() - 1 { + return compilation_error( + lines, + line, + format!( + "invalid reference \\{} on `s' command's RHS", + subst.replacement.max_group_number + ), + ); + } } cmd.data = CommandData::Substitution(subst); @@ -1153,6 +1167,7 @@ fn lookup_command(cmd: char) -> Option<&'static CommandSpec> { #[cfg(test)] mod tests { use super::*; + use crate::fast_io::IOChunk; // Return an empty line provider and a char provider for the specified str. fn make_providers(input: &str) -> (ScriptLineProvider, ScriptCharProvider) { @@ -1370,8 +1385,8 @@ mod tests { let regex = compile_regex(&lines, &chars, "abc", &ctx(), false) .unwrap() .expect("regex should be present"); - assert!(regex.is_match("abc").unwrap()); - assert!(!regex.is_match("ABC").unwrap()); + assert!(regex.is_match(&mut IOChunk::from_str("abc")).unwrap()); + assert!(!regex.is_match(&mut IOChunk::from_str("ABC")).unwrap()); } #[test] @@ -1380,9 +1395,9 @@ mod tests { let regex = compile_regex(&lines, &chars, "abc", &ctx(), true) .unwrap() .expect("regex should be present"); - assert!(regex.is_match("abc").unwrap()); - assert!(regex.is_match("ABC").unwrap()); - assert!(regex.is_match("AbC").unwrap()); + assert!(regex.is_match(&mut IOChunk::from_str("abc")).unwrap()); + assert!(regex.is_match(&mut IOChunk::from_str("ABC")).unwrap()); + assert!(regex.is_match(&mut IOChunk::from_str("AbC")).unwrap()); } #[test] @@ -1430,7 +1445,7 @@ mod tests { let addr = compile_address(&lines, &mut chars, &ctx()).unwrap(); assert!(matches!(addr.atype, AddressType::Re)); if let AddressValue::Regex(Some(re)) = addr.value { - assert!(re.is_match("hello").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("hello")).unwrap()); } else { panic!("expected Regex address value"); } @@ -1442,7 +1457,7 @@ mod tests { let addr = compile_address(&lines, &mut chars, &ctx()).unwrap(); assert!(matches!(addr.atype, AddressType::Re)); if let AddressValue::Regex(Some(re)) = addr.value { - assert!(re.is_match("hello").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("hello")).unwrap()); } else { panic!("expected Regex address value"); } @@ -1454,7 +1469,7 @@ mod tests { let addr = compile_address(&lines, &mut chars, &ctx()).unwrap(); assert!(matches!(addr.atype, AddressType::Re)); if let AddressValue::Regex(Some(re)) = addr.value { - assert!(!re.is_match("helio").unwrap()); + assert!(!re.is_match(&mut IOChunk::from_str("helio")).unwrap()); } else { panic!("expected Regex address value"); } @@ -1466,7 +1481,7 @@ mod tests { let addr = compile_address(&lines, &mut chars, &ctx()).unwrap(); assert!(matches!(addr.atype, AddressType::Re)); if let AddressValue::Regex(Some(re)) = addr.value { - assert!(re.is_match("hello").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("hello")).unwrap()); } else { panic!("expected Regex address value"); } @@ -1478,7 +1493,7 @@ mod tests { let addr = compile_address(&lines, &mut chars, &ctx()).unwrap(); assert!(matches!(addr.atype, AddressType::Re)); if let AddressValue::Regex(Some(re)) = addr.value { - assert!(re.is_match("HELLO").unwrap()); // case-insensitive + assert!(re.is_match(&mut IOChunk::from_str("HELLO")).unwrap()); // case-insensitive } else { panic!("expected Regex address value"); } @@ -1569,8 +1584,8 @@ mod tests { AddressType::Re )); if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr1.as_ref().unwrap().value { - assert!(re.is_match("foo").unwrap()); - assert!(!re.is_match("bar").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("foo")).unwrap()); + assert!(!re.is_match(&mut IOChunk::from_str("bar")).unwrap()); } else { panic!("expected a regex address"); }; @@ -1589,8 +1604,8 @@ mod tests { AddressType::Re )); if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr1.as_ref().unwrap().value { - assert!(re.is_match("foo").unwrap()); - assert!(!re.is_match("bar").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("foo")).unwrap()); + assert!(!re.is_match(&mut IOChunk::from_str("bar")).unwrap()); } else { panic!("expected a regex address"); } @@ -1600,8 +1615,8 @@ mod tests { AddressType::Re )); if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr2.as_ref().unwrap().value { - assert!(re.is_match("bar").unwrap()); - assert!(!re.is_match("foo").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("bar")).unwrap()); + assert!(!re.is_match(&mut IOChunk::from_str("foo")).unwrap()); } else { panic!("expected a regex address"); }; @@ -1619,8 +1634,8 @@ mod tests { AddressType::Re )); if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr1.as_ref().unwrap().value { - assert!(re.is_match("FOO").unwrap()); - assert!(re.is_match("foo").unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("FOO")).unwrap()); + assert!(re.is_match(&mut IOChunk::from_str("foo")).unwrap()); } else { panic!("expected a regex address with case-insensitive match"); }; @@ -1796,6 +1811,18 @@ mod tests { assert!(matches!(&template.parts[1], ReplacementPart::WholeMatch)); } + #[test] + fn test_compile_replacement_whole_match_synonym() { + let (mut lines, mut chars) = make_providers(r"/The match was: \0/"); + let template = compile_replacement(&mut lines, &mut chars).unwrap(); + + assert_eq!(template.parts.len(), 2); + assert!( + matches!(&template.parts[0], ReplacementPart::Literal(s) if s == "The match was: ") + ); + assert!(matches!(&template.parts[1], ReplacementPart::WholeMatch)); + } + #[test] fn test_compile_replacement_ampersand() { let (mut lines, mut chars) = make_providers("/Simon \\& Garfunkel/"); @@ -1989,6 +2016,15 @@ mod tests { } } + #[test] + fn test_compile_subst_invalid_group_reference() { + let (mut lines, mut chars) = make_providers(r"s/f(o)o/\2/"); + let mut cmd = Command::default(); + + let err = compile_subst_command(&mut lines, &mut chars, &mut cmd, &ctx()).unwrap_err(); + assert!(err.to_string().contains("invalid reference \\2")); + } + // bre_to_ere #[test] fn test_bre_group_translation() { diff --git a/src/uu/sed/src/fast_io.rs b/src/uu/sed/src/fast_io.rs index 36bb53e..273e667 100644 --- a/src/uu/sed/src/fast_io.rs +++ b/src/uu/sed/src/fast_io.rs @@ -14,9 +14,12 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +#[cfg(unix)] +use memchr::memchr; #[cfg(unix)] use memmap2::Mmap; +use std::cell::Cell; use std::fs::File; use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; @@ -69,10 +72,12 @@ impl<'a> MmapLineCursor<'a> { } let start = self.pos; - let mut end = start; - while end < self.data.len() && self.data[end] != b'\n' { - end += 1; - } + + let mut end = if let Some(pos) = memchr(b'\n', &self.data[start..]) { + pos + start + } else { + self.data.len() + }; if end < self.data.len() { end += 1; // include \n in full span @@ -135,7 +140,7 @@ impl ReadLineCursor { /// As chunk of data that is input and can be output, often very efficiently #[derive(Debug, PartialEq, Eq)] pub struct IOChunk<'a> { - utf8_verified: bool, // True if the contents are valid UTF-8 + utf8_verified: Cell, // True if the contents are valid UTF-8 content: IOChunkContent<'a>, } @@ -143,14 +148,14 @@ impl<'a> IOChunk<'a> { /// Construct an IOChunk from the given content fn from_content(content: IOChunkContent<'a>) -> Self { Self { - utf8_verified: false, + utf8_verified: Cell::new(false), content, } } /// Clear the object's contents, converting it into Owned if needed. pub fn clear(&mut self) { - self.utf8_verified = true; + self.utf8_verified.set(true); match &mut self.content { IOChunkContent::Owned { content, @@ -182,10 +187,19 @@ impl<'a> IOChunk<'a> { } } + #[cfg(test)] + /// Create an Owned newline-terminated IOChunk from a string. + pub fn from_str(s: &str) -> Self { + IOChunk { + content: IOChunkContent::new_owned(s.to_string(), true), + utf8_verified: Cell::new(false), + } + } + /// Set the object's contents to the specified string. /// Convert it into Owned if needed. pub fn set_to_string(&mut self, new_content: String, add_newline: bool) { - self.utf8_verified = true; + self.utf8_verified.set(true); match &mut self.content { IOChunkContent::Owned { content, @@ -203,16 +217,16 @@ impl<'a> IOChunk<'a> { } /// Return the content as a str. - pub fn as_str(&mut self) -> Result<&str, Box> { + pub fn as_str(&self) -> Result<&str, Box> { match &self.content { #[cfg(unix)] IOChunkContent::MmapInput { content, .. } => { - if self.utf8_verified { + if self.utf8_verified.get() { // Use cached result Ok(unsafe { self.content.as_str_unchecked() }) } else { let result = str::from_utf8(content); - self.utf8_verified = true; + self.utf8_verified.set(true); result.map_err(|e| USimpleError::new(1, e.to_string())) } } @@ -220,6 +234,15 @@ impl<'a> IOChunk<'a> { } } + /// Return the raw byte content (always safe). + pub fn as_bytes(&self) -> &[u8] { + match &self.content { + #[cfg(unix)] + IOChunkContent::MmapInput { content, .. } => content, + IOChunkContent::Owned { content, .. } => content.as_bytes(), + } + } + /// Convert content to the Owned variant if it's not already. /// Fails if the conversion to UTF-8 fails. pub fn ensure_owned(&mut self) -> Result<(), Box> { @@ -232,7 +255,7 @@ impl<'a> IOChunk<'a> { let has_newline = full_span.last().copied() == Some(b'\n'); self.content = IOChunkContent::new_owned(valid_str.to_string(), has_newline); - self.utf8_verified = true; + self.utf8_verified.set(true); Ok(()) } Err(e) => Err(USimpleError::new(1, e.to_string())), @@ -934,7 +957,7 @@ mod tests { { assert_eq!(content, "first line"); assert!(has_newline); - assert!(!utf8_verified); + assert!(!utf8_verified.get()); assert!(!last_line); } else { panic!("Expected IOChunkContent::Owned"); @@ -960,7 +983,7 @@ mod tests { panic!("Expected IOChunkContent::Owned"); } - if let Some((mut content, last_line)) = reader.get_line()? { + if let Some((content, last_line)) = reader.get_line()? { assert_eq!(content.as_str().unwrap(), "last line"); assert!(last_line); } else { @@ -998,7 +1021,7 @@ mod tests { { assert_eq!(content, b"first line"); assert_eq!(full_span, b"first line\n"); - assert!(!utf8_verified); + assert!(!utf8_verified.get()); assert!(!last_line); } else { panic!("Expected IOChunkContent::MapInput"); @@ -1018,15 +1041,16 @@ mod tests { { assert_eq!(content, b"second line"); assert_eq!(full_span, b"second line\n"); - assert!(!utf8_verified); + assert!(!utf8_verified.get()); assert!(!last_line); } else { panic!("Expected IOChunkContent::MapInput"); } - if let Some((mut content, last_line)) = reader.get_line()? { + if let Some((content, last_line)) = reader.get_line()? { + assert_eq!(content.as_bytes(), b"last line"); assert_eq!(content.as_str().unwrap(), "last line"); - assert!(content.utf8_verified); + assert!(content.utf8_verified.get()); assert!(last_line); // Cached version assert_eq!(content.as_str().unwrap(), "last line"); diff --git a/src/uu/sed/src/fast_regex.rs b/src/uu/sed/src/fast_regex.rs new file mode 100644 index 0000000..b45d36c --- /dev/null +++ b/src/uu/sed/src/fast_regex.rs @@ -0,0 +1,745 @@ +// A unified interface to byte and fancy Regex +// +// This allows using byte Regex when possible, resorting to the +// slower fancy_regex crate when needed. +// +// SPDX-License-Identifier: MIT +// Copyright (c) 2025 Diomidis Spinellis +// +// This file is part of the uutils sed package. +// It is licensed under the MIT License. +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use fancy_regex::{ + CaptureMatches as FancyCaptureMatches, Captures as FancyCaptures, Regex as FancyRegex, +}; +use memchr::memmem; +use once_cell::sync::Lazy; +use regex::Regex as RustRegex; +use regex::bytes::{ + CaptureMatches as ByteCaptureMatches, Captures as ByteCaptures, Regex as ByteRegex, +}; +use std::error::Error; +use uucore::error::{UResult, USimpleError}; + +use crate::fast_io::IOChunk; + +/// REs requiring the fancy_regex capabilities rather than the +/// faster regex::bytes engine +// Consider . as one character that requires fancy_regex, +// because it can match more than one byte when matching a +// two or more byte Unicode UTF-8 representation. +// It is an RE . rather than a literal one in the following +// example sitations. +// . First character of the line +// [^\\]. Second character after non \ +// +// \*. A consumed backslash anywhere on the line +// \\. An escaped backslash anywhere on the line +// xx. A non-escaped sequence anywhere on the line +// But the following are literal dots and can be captured by bytes: +// \. escaped at the beginning of the line +// x\. escaped after a non escaped \ anywhere on the line +// +// The following RE captures these situations. +static NEEDS_FANCY_RE: Lazy = Lazy::new(|| { + regex::Regex::new( + r"(?x) # Turn on verbose mode + ( # An ASCII-incompatible RE + ( ^ # Non-escaped: i.e. at BOL + | ^[^\\] # or after a BOL non \ + | [^\\] {2} # or after two non \ characters + | \\. # or after a consumed or escaped \ + ) + ( # A potentially incompatible match + \. # . matches any Unicode character + | \[\^ # Bracketed -ve character class + | \(\?i # (Unicode) case insensitive + | \\[WwDdSsBbPp] # Unicode classes + | \\[0-9] # Back-references need fancy + ) + ) + | [^\x01-\x7f] # Any non-ASCII character + ", + ) + .unwrap() +}); + +/// All characters signifying that the match must be handled by an RE +/// rather than by plain string pattern matching. +// These do not include the ^$ metacharacters, which we can easily handle. +// Plain string fixed-string matching is currently faster than Regex +// matching, because Regex always constructs an automaton and needs +// to handle state transitions, whereas plain string matching can +// use tailored CPU string or vectored instructions. +static NEEDS_RE: Lazy = Lazy::new(|| { + regex::Regex::new( + r"(?x) # Turn on verbose mode + ( ^ # Non-escaped: i.e. at BOL + | ^[^\\] # or after a BOL non \ + | [^\\] {2} # or after two non \ characters + | \\. # or after a consumed or escaped \ + ) + ( # A potentially incompatible match + [.?|+(\[{*] # Any magic RE character + # Some are operators so illegal at + # BOL but they should error there, + # not use them as literals. + | \\[WwDdSsPp] # Unicode classes + | \\[AzBb] # Empty matches + | \\[0-9] # Back-references + ) + ", + ) + .unwrap() +}); + +#[derive(Clone, Debug)] +/// Types of literal string anchored matches +enum AnchoredMatch { + Begin, // ^... + End, // ...$ + Both, // ^...$ + Free, // ... +} + +#[derive(Clone, Debug)] +/// A fast Regex-like matcher for literal strings using memchr:memmem +pub struct LiteralMatcher { + needle: Vec, // Bytes without any anchors + match_type: AnchoredMatch, // Type of anchoring specified +} + +impl LiteralMatcher { + /// Construct a new matcher based on a needle possible with anchors. + pub fn new(needle: &str) -> Self { + let needle_bytes = needle.as_bytes(); + if needle_bytes[0] == b'^' && needle_bytes[needle_bytes.len() - 1] == b'$' { + LiteralMatcher { + match_type: AnchoredMatch::Both, + needle: needle_bytes[1..needle_bytes.len() - 1].to_vec(), + } + } else if needle_bytes[0] == b'^' { + LiteralMatcher { + match_type: AnchoredMatch::Begin, + needle: needle_bytes[1..needle_bytes.len()].to_vec(), + } + } else if needle_bytes[needle_bytes.len() - 1] == b'$' { + LiteralMatcher { + match_type: AnchoredMatch::End, + needle: needle_bytes[0..needle_bytes.len() - 1].to_vec(), + } + } else { + LiteralMatcher { + match_type: AnchoredMatch::Free, + needle: needle_bytes.to_vec(), + } + } + } + + /// Returns the start index of a match, if any + fn anchored_find(&self, haystack: &[u8]) -> Option { + let nlen = self.needle.len(); + let hlen = haystack.len(); + + match self.match_type { + AnchoredMatch::Both => { + if hlen == nlen && haystack == self.needle.as_slice() { + Some(0) + } else { + None + } + } + AnchoredMatch::Begin => { + if hlen >= nlen && &haystack[..nlen] == self.needle.as_slice() { + Some(0) + } else { + None + } + } + AnchoredMatch::End => { + if hlen >= nlen && &haystack[hlen - nlen..] == self.needle.as_slice() { + Some(hlen - nlen) + } else { + None + } + } + AnchoredMatch::Free => memmem::find(haystack, &self.needle), + } + } + + /// Return true if the needle occurs in the haystack. + pub fn is_match(&self, haystack: &[u8]) -> bool { + self.anchored_find(haystack).is_some() + } + + /// Return the position and contents of the matched needle. + pub fn find<'t>(&self, haystack: &'t [u8]) -> Option<(usize, usize, &'t str)> { + self.anchored_find(haystack).and_then(|start| { + let end = start + self.needle.len(); + std::str::from_utf8(&haystack[start..end]) + .ok() + .map(|s| (start, end, s)) + }) + } + + /// Return all positions and contents of the matched needle. + pub fn iter<'t>( + &'t self, + haystack: &'t [u8], + ) -> Box + 't> { + let needle = &self.needle; + let nlen = needle.len(); + + match self.match_type { + AnchoredMatch::Both | AnchoredMatch::Begin | AnchoredMatch::End => { + // At most one match; yield it if present + Box::new(self.find(haystack).into_iter()) + } + AnchoredMatch::Free => { + // Multiple potential matches + Box::new( + memmem::find_iter(haystack, needle).filter_map(move |start| { + let end = start + nlen; + std::str::from_utf8(&haystack[start..end]) + .ok() + .map(|s| (start, end, s)) + }), + ) + } + } + } +} + +/// Return the passed pattern without any backslash escapes. +pub fn remove_escapes(pattern: &str) -> String { + let mut chars = pattern.chars().peekable(); + let mut result = String::with_capacity(pattern.len()); + + while let Some(c) = chars.next() { + if c == '\\' { + // Look ahead and consume the next character if present + if let Some(&next) = chars.peek() { + result.push(next); + chars.next(); // consume the peeked char + } + } else { + result.push(c); + } + } + + result +} + +#[derive(Clone, Debug)] +/// A regular expression that can be implemented in diverse efficient ways +pub enum Regex { + Literal(LiteralMatcher), // Fastest: literal bytes + Byte(ByteRegex), // Slower: byte-based RE + Fancy(FancyRegex), // Slowest: RE supporting UTF-8 and back-references +} + +impl Regex { + /// Construct the most efficient RE-like matching engine possible. + pub fn new(pattern: &str) -> Result> { + if NEEDS_FANCY_RE.is_match(pattern) { + Ok(Self::Fancy(FancyRegex::new(pattern)?)) + } else if NEEDS_RE.is_match(pattern) { + Ok(Self::Byte(ByteRegex::new(pattern)?)) + } else { + Ok(Self::Literal(LiteralMatcher::new(&remove_escapes(pattern)))) + } + } + + /// Check if the regex matches the content of the IOChunk. + pub fn is_match(&self, chunk: &mut IOChunk) -> UResult { + match self { + Regex::Literal(m) => Ok(m.is_match(chunk.as_bytes())), + Regex::Byte(re) => Ok(re.is_match(chunk.as_bytes())), + Regex::Fancy(re) => { + let text = chunk.as_str()?; + re.is_match(text) + .map_err(|e| USimpleError::new(2, e.to_string())) + } + } + } + + /// Return an iterator over capture groups. + pub fn captures_iter<'t>(&'t self, chunk: &'t IOChunk) -> UResult> { + match self { + Regex::Literal(m) => { + let haystack = chunk.as_bytes(); + Ok(CaptureMatches::Literal(Box::new(m.iter(haystack).map( + |(start, end, text)| Ok(Captures::Literal(Match { start, end, text })), + )))) + } + + Regex::Byte(re) => Ok(CaptureMatches::Byte(re.captures_iter(chunk.as_bytes()))), + + Regex::Fancy(re) => { + let text = chunk.as_str()?; + Ok(CaptureMatches::Fancy(re.captures_iter(text))) + } + } + } + + /// Return the number of capture groups, including group 0. + pub fn captures_len(&self) -> usize { + match self { + Regex::Literal(_) => 1, // Only group 0 + Regex::Byte(re) => re.captures_len(), + Regex::Fancy(re) => re.captures_len(), + } + } + + /// Return the elements of the first capture. + pub fn captures<'t>(&self, chunk: &'t IOChunk) -> UResult>> { + match self { + Regex::Literal(m) => { + let haystack = chunk.as_bytes(); + match m.find(haystack) { + Some((start, end, text)) => { + Ok(Some(Captures::Literal(Match { start, end, text }))) + } + None => Ok(None), + } + } + + Regex::Byte(re) => { + let bytes = chunk.as_bytes(); + Ok(re.captures(bytes).map(Captures::Byte)) + } + + Regex::Fancy(re) => { + let text = chunk.as_str()?; + match re.captures(text) { + Ok(Some(caps)) => Ok(Some(Captures::Fancy(caps))), + Ok(None) => Ok(None), + Err(e) => Err(USimpleError::new(2, e.to_string())), + } + } + } + } + + /// Return a non-capturing result for a single match. + pub fn find<'t>(&self, chunk: &'t IOChunk) -> UResult>> { + match self { + Regex::Literal(m) => { + let haystack = chunk.as_bytes(); + match m.find(haystack) { + Some((start, end, text)) => Ok(Some(Match { start, end, text })), + None => Ok(None), + } + } + + Regex::Byte(re) => { + let haystack = chunk.as_bytes(); + if let Some(m) = re.find(haystack) { + // Attempt UTF-8 decode for the match region only + let text = std::str::from_utf8(&haystack[m.start()..m.end()]) + .map_err(|e| USimpleError::new(2, e.to_string()))?; + Ok(Some(Match { + start: m.start(), + end: m.end(), + text, + })) + } else { + Ok(None) + } + } + + Regex::Fancy(re) => { + let text = chunk.as_str()?; + match re.find(text) { + Ok(Some(m)) => Ok(Some(Match { + start: m.start(), + end: m.end(), + text: m.as_str(), + })), + Ok(None) => Ok(None), + Err(e) => Err(USimpleError::new(2, e.to_string())), + } + } + } + } +} + +/// Unified enum for holding either byte or fancy capture iterators. +pub enum CaptureMatches<'t> { + Literal(Box>> + 't>), + Byte(ByteCaptureMatches<'t, 't>), + Fancy(FancyCaptureMatches<'t, 't>), +} + +impl<'t> Iterator for CaptureMatches<'t> { + type Item = UResult>; + + fn next(&mut self) -> Option { + match self { + CaptureMatches::Literal(iter) => iter.next(), + CaptureMatches::Byte(iter) => iter.next().map(|caps| Ok(Captures::Byte(caps))), + CaptureMatches::Fancy(iter) => match iter.next() { + Some(Ok(caps)) => Some(Ok(Captures::Fancy(caps))), + Some(Err(e)) => Some(Err(USimpleError::new( + 2, + format!("error retrieving RE captures: {}", e), + ))), + None => None, + }, + } + } +} + +#[derive(Clone, Debug)] +/// Result type for RE capture get(n) +pub struct Match<'t> { + start: usize, // Match start + end: usize, // Match end + text: &'t str, // Actual match +} + +/// Provide interface compatible with Regex::Match. +impl<'t> Match<'t> { + pub fn start(&self) -> usize { + self.start + } + + pub fn end(&self) -> usize { + self.end + } + + pub fn as_str(&self) -> &'t str { + self.text + } +} + +/// Provide interface compatible with Regex::Captures. +pub enum Captures<'t> { + Literal(Match<'t>), // only group 0 + Byte(ByteCaptures<'t>), + Fancy(FancyCaptures<'t>), +} + +impl<'t> Captures<'t> { + /// Get capture group at index `i` + /// Returns Ok(None) if the group didn't match. + /// Returns Err if UTF-8 conversion fails (in Byte variant). + pub fn get(&self, i: usize) -> UResult>> { + match self { + Captures::Literal(m) => Ok(if i == 0 { Some(m.clone()) } else { None }), + Captures::Byte(caps) => match caps.get(i) { + Some(m) => Ok(Some(Match { + start: m.start(), + end: m.end(), + text: std::str::from_utf8(m.as_bytes()) + .map_err(|e| USimpleError::new(1, e.to_string()))?, + })), + None => Ok(None), + }, + Captures::Fancy(caps) => match caps.get(i) { + Some(m) => Ok(Some(Match { + start: m.start(), + end: m.end(), + text: m.as_str(), + })), + None => Ok(None), + }, + } + } + + /// Return the number of capture groups (including group 0). + pub fn len(&self) -> usize { + match self { + Captures::Literal(_) => 1, + Captures::Byte(caps) => caps.len(), + Captures::Fancy(caps) => caps.len(), + } + } + + /// Return true if there are no captures. + // Unused, but provided for completeness. + pub fn is_empty(&self) -> bool { + match self { + Captures::Literal(_) => false, // A literal match always has group 0 + Captures::Byte(caps) => caps.len() == 0, + Captures::Fancy(caps) => caps.len() == 0, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // FANCY_RE + #[test] + fn test_needs_fancy_re_matches() { + let should_match = [ + // Unicode classes BOL + r"\p{L}+", // Unicode letter class + r"\W", // \W is Unicode-aware. + r"\S+", // \S is Unicode-aware. + r"\d", // \d includes all Unicode digits. + // Unicode classes non-BOL + r"x\p{L}+", // Unicode letter class + r"x\W", // \W is Unicode-aware. + r"x\S+", // \S is Unicode-aware. + r"x\d", // \d includes all Unicode digits. + // . + r".", + r"x.", + r"xx.", + // Consumed \ + r"\*.", + r"x\*.", + // Escaped \ + r"\\.", + r"x\\.", + // Inline flags + r"(?i)abc", // Unicode case-insensitive + r"x(?i)abc", // Unicode case-insensitive + r"(\w+):\1", // back-reference \1 + // Non-ASCII literals + "naïve", // Contains literal non-ASCII. + "café", // Contains literal non-ASCII. + ]; + + for pat in &should_match { + assert!( + NEEDS_FANCY_RE.is_match(pat), + "Expected NEEDS_FANCY_RE to match: {:?}", + pat + ); + } + } + + #[test] + fn test_needs_fancy_re_does_not_match() { + let should_not_match = [ + r"\.", // Escaped . at BOL + r"x\.", // Escaped . at non BOL + r"\[^x]", // Escaped character class + r"\(?i\)", // Escaped case insesitive flag + r"\\w", // Escaped Unicode class + // Simple ASCII + r"foo", + r"foo|bar", + r"^foo[0-9]+bar$", + ]; + + for pat in &should_not_match { + assert!( + !NEEDS_FANCY_RE.is_match(pat), + "Expected NEEDS_FANCY_RE to NOT match: {:?}", + pat + ); + } + } + + // NEEDS_RE + #[test] + fn test_needs_re_matches() { + let should_match = [ + r".", // Single regex wildcard + r"a+b", // Regex + + r"foo|bar", // Regex alternation + r"abc?", // Regex optional + r"a*b", // Regex star + r"[abc]", // Character class + r"(abc)", // Group + r"{1,2}", // Repetition + r"\d", // Class shorthand + r"\S", // Class shorthand + r"\1", // Backreference + r"a\Pb", // Unicode property + ]; + + for pat in &should_match { + assert!( + NEEDS_RE.is_match(pat), + "Expected NEEDS_RE to match: {:?}", + pat + ); + } + } + + #[test] + fn test_needs_re_does_not_match() { + let should_not_match = [ + r"abc", + r"a\.b", // Escaped dot + r"hello world", + r"^abc$", // Anchors alone + r"file\.", // Escaped dot + r"literal123", + r"\\", // Escaped backslash + ]; + + for pat in &should_not_match { + assert!( + !NEEDS_RE.is_match(pat), + "Expected NEEDS_RE to NOT match: {:?}", + pat + ); + } + } + + // Regex::new + #[test] + fn assert_byte_selection() { + let re = Regex::new(r"x*").unwrap(); + assert!(matches!(re, Regex::Byte(_))); + } + + #[test] + fn assert_fancy() { + let re = Regex::new(r"\d").unwrap(); + assert!(matches!(re, Regex::Fancy(_))); + } + + #[test] + fn assert_literal() { + let re = Regex::new(r"x\.").unwrap(); + assert!(matches!(re, Regex::Literal(_))); + } + + #[test] + fn handles_invalid_regex_gracefully() { + let err = Regex::new("(").unwrap_err().to_string(); + assert!( + err.contains("unclosed group") || err.contains("error parsing"), + "Unexpected error: {}", + err + ); + } + + // remove_escapes + #[test] + fn test_remove_escapes() { + use super::remove_escapes; + + assert_eq!(remove_escapes("abc"), "abc"); + assert_eq!(remove_escapes(r"a\.c"), "a.c"); + assert_eq!(remove_escapes(r"\\d"), r"\d"); + assert_eq!(remove_escapes(r"\.\*\+\?"), ".*+?"); + assert_eq!(remove_escapes(r"escaped\\backslash"), r"escaped\backslash"); + assert_eq!(remove_escapes(r"trailing\\"), r"trailing\"); + } + + // LiteralMatcher + #[test] + fn test_literal_matcher_basic_match() { + let matcher = LiteralMatcher::new("needle"); + assert!(matcher.is_match(b"this is a needle in a haystack")); + assert!(!matcher.is_match(b"no match here")); + } + + #[test] + fn test_literal_matcher_anchor_start_match() { + let matcher = LiteralMatcher::new("^needle"); + assert!(matcher.is_match(b"needle in a haystack")); + assert!(!matcher.is_match(b"no needle match here")); + assert!(!matcher.is_match(b"no")); + } + + #[test] + fn test_literal_matcher_anchor_end_match() { + let matcher = LiteralMatcher::new("needle$"); + assert!(matcher.is_match(b"In a haystack there's a needle")); + assert!(!matcher.is_match(b"no needle match here")); + assert!(!matcher.is_match(b"no")); + } + + #[test] + fn test_literal_matcher_anchor_begin_end_match() { + let matcher = LiteralMatcher::new("^needle$"); + assert!(matcher.is_match(b"needle")); + assert!(!matcher.is_match(b"no needle match")); + assert!(!matcher.is_match(b"needle no match")); + assert!(!matcher.is_match(b"no match needle")); + assert!(!matcher.is_match(b"nada")); + } + + #[test] + fn test_literal_matcher_utf8_match() { + let matcher = LiteralMatcher::new("✓"); // U+2713 CHECK MARK (3 bytes) + let haystack = "contains ✓ unicode".as_bytes(); + assert!(matcher.is_match(haystack)); + let found = matcher.find(haystack).unwrap(); + assert_eq!(found.2, "✓"); + } + + #[test] + fn test_literal_matcher_find_location() { + let matcher = LiteralMatcher::new("abc"); + let haystack = b"___abc___"; + let result = matcher.find(haystack); + assert!(result.is_some()); + let (start, end, text) = result.unwrap(); + assert_eq!((start, end), (3, 6)); + assert_eq!(text, "abc"); + } + + #[test] + fn test_literal_matcher_find_location_end() { + let matcher = LiteralMatcher::new("abc$"); + let haystack = b"012abc"; + let result = matcher.find(haystack); + assert!(result.is_some()); + let (start, end, text) = result.unwrap(); + assert_eq!((start, end), (3, 6)); + assert_eq!(text, "abc"); + } + + #[test] + fn test_literal_matcher_iter_multiple() { + let matcher = LiteralMatcher::new("test"); + let haystack = b"this test is a test of test matching"; + let matches: Vec<_> = matcher.iter(haystack).collect(); + assert_eq!(matches.len(), 3); + + let strings: Vec<_> = matches.iter().map(|(_, _, s)| *s).collect(); + assert_eq!(strings, ["test", "test", "test"]); + } + + #[test] + fn test_literal_matcher_iter_begin() { + let matcher = LiteralMatcher::new("^test"); + let haystack = b"test is a test of test matching"; + let matches: Vec<_> = matcher.iter(haystack).collect(); + assert_eq!(matches.len(), 1); + + let strings: Vec<_> = matches.iter().map(|(_, _, s)| *s).collect(); + assert_eq!(strings, ["test"]); + } + + #[test] + fn test_literal_matcher_iter_end() { + let matcher = LiteralMatcher::new("test$"); + let haystack = b"this test is a test of test"; + let matches: Vec<_> = matcher.iter(haystack).collect(); + assert_eq!(matches.len(), 1); + + let strings: Vec<_> = matches.iter().map(|(_, _, s)| *s).collect(); + assert_eq!(strings, ["test"]); + } + + #[test] + fn test_literal_matcher_no_match() { + let matcher = LiteralMatcher::new("missing"); + let haystack = b"nothing to see here"; + assert!(!matcher.is_match(haystack)); + assert!(matcher.find(haystack).is_none()); + assert_eq!(matcher.iter(haystack).count(), 0); + } + + #[test] + fn test_literal_matcher_anchored_no_match() { + let matcher = LiteralMatcher::new("^see$"); + let haystack = b"nothing to see here"; + assert!(!matcher.is_match(haystack)); + assert!(matcher.find(haystack).is_none()); + assert_eq!(matcher.iter(haystack).count(), 0); + } +} diff --git a/src/uu/sed/src/processor.rs b/src/uu/sed/src/processor.rs index 7ac819b..24253a2 100644 --- a/src/uu/sed/src/processor.rs +++ b/src/uu/sed/src/processor.rs @@ -13,9 +13,10 @@ use crate::command::{ ProcessingContext, Substitution, Transliteration, }; use crate::fast_io::{IOChunk, LineReader, OutputBuffer}; +use crate::fast_regex::Regex; use crate::in_place::InPlace; use crate::named_writer; -use fancy_regex::Regex; + use std::cell::RefCell; use std::io::{self, IsTerminal}; use std::path::PathBuf; @@ -32,10 +33,7 @@ fn match_address( AddressType::Re => { if let AddressValue::Regex(ref re) = addr.value { let regex = re_or_saved_re(re, context)?; - let matched = regex.is_match(pattern.as_str()?).map_err(|e| { - USimpleError::new(2, format!("regular expression match error: {}", e)) - })?; - Ok(matched) + regex.is_match(pattern) } else { Ok(false) } @@ -197,39 +195,78 @@ fn substitute( let mut result = String::new(); let mut replaced = false; - let text = pattern.as_str()?; + let mut text: Option<&str> = None; let regex = re_or_saved_re(&sub.regex, context)?; - for caps in regex.captures_iter(text) { - count += 1; - let caps = caps.map_err(|e| { - USimpleError::new( - 2, - format!("regular expression capture retrieval error: {}", e), - ) - })?; + match (sub.occurrence, sub.replacement.max_group_number) { + (1, 0) => { + // Example: s/foo/bar/: find() is enough. + let m = regex.find(pattern)?; + if let Some(m) = m { + text = Some(pattern.as_str()?); + result.push_str(&text.unwrap()[last_end..m.start()]); - let m = caps.get(0).unwrap(); - - // Always write the unmatched text before this match. - result.push_str(&text[last_end..m.start()]); - - if sub.occurrence == 0 || count == sub.occurrence { - let replacement = sub.replacement.apply(&caps)?; - result.push_str(&replacement); - replaced = true; - } else { - // Not the target match — leave the match unchanged. - result.push_str(m.as_str()); + let replacement = sub.replacement.apply_match(&m); + result.push_str(&replacement); + replaced = true; + last_end = m.end(); + } } - last_end = m.end(); + (1, _) => { + // Example: s/\(.\)\(.\)/\2\1/: captures() is enough. + let caps = regex.captures(pattern)?; + if let Some(caps) = caps { + let m = caps.get(0)?.unwrap(); + text = Some(pattern.as_str()?); + result.push_str(&text.unwrap()[last_end..m.start()]); + + let replacement = sub.replacement.apply_captures(&caps)?; + result.push_str(&replacement); + replaced = true; + last_end = m.end(); + } + } + + (_, _) => { + // Example: s/(.)(.)/\2\1/3: captures_iter() is needed. + // Iterate over multiple captures of the RE in the pattern. + for caps in regex.captures_iter(pattern)? { + count += 1; + let caps = caps?; + + let m = caps.get(0)?.unwrap(); + + // Always write the unmatched text before this match. + if text.is_none() { + text = Some(pattern.as_str()?); + } + result.push_str(&text.unwrap()[last_end..m.start()]); + + if sub.occurrence == 0 || count == sub.occurrence { + let replacement = sub.replacement.apply_captures(&caps)?; + result.push_str(&replacement); + replaced = true; + } else { + // Not the target match — leave the match unchanged. + result.push_str(m.as_str()); + } + + last_end = m.end(); + + // Early exit if only a specific occurrence, + // (likely 1) needed replacing. + if count == sub.occurrence { + break; + } + } + } } // Handle substitution success. if replaced { - result.push_str(&text[last_end..]); + result.push_str(&text.unwrap()[last_end..]); pattern.set_to_string(result, pattern.is_newline_terminated()); diff --git a/src/uu/sed/src/sed.rs b/src/uu/sed/src/sed.rs index 13f3a1b..d61bc96 100644 --- a/src/uu/sed/src/sed.rs +++ b/src/uu/sed/src/sed.rs @@ -12,6 +12,7 @@ pub mod command; pub mod compiler; pub mod delimited_parser; pub mod fast_io; +pub mod fast_regex; pub mod in_place; pub mod named_writer; pub mod processor; diff --git a/tests/fixtures/sed/script/http-log-redact.sed b/tests/fixtures/sed/script/http-log-redact.sed new file mode 100644 index 0000000..600ae9f --- /dev/null +++ b/tests/fixtures/sed/script/http-log-redact.sed @@ -0,0 +1,4 @@ +#!/ +s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[REDACTED_IP]/; +s/\[[0-9]{2}\/[A-Za-z]{3}\/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} [+-][0-9]{4}\]/[TIMESTAMP]/; +s/"Mozilla[^"]*"/"[UA]"/ diff --git a/util/bench-compare.sh b/util/bench-compare.sh new file mode 100755 index 0000000..7e829c6 --- /dev/null +++ b/util/bench-compare.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# +# Compare the results of two benchmarks reporting fastest for each task +# + + +set -eu + +if [ $# -ne 2 ] ; then + echo "Usage: $(basename $0) results-1 results-2" 1>&2 + + exit 1 +fi + +paste -d , "$1" "$2" | + sed "s/^/$1,$2,/" | + + awk -F, 'FNR != 1 { + printf("%22s ", $3); + if (!$4) + printf("%12s encountered an error\n", $1); + else if (!$12) + printf("%12s encountered an error\n", $2); + else if ($4 / $12 > 1.01) + printf("%12s is %5.2f times faster than %s\n", $2, $4 / $12, $1); + else if ($12 / $4 > 1.01) + printf("%12s is %5.2f times faster than %s\n", $1, $12 / $4, $2); + else + printf("%12s is similarly fast as %s\n", $1, $2); + }' diff --git a/util/benchmark.sh b/util/benchmark.sh new file mode 100755 index 0000000..ebbb02a --- /dev/null +++ b/util/benchmark.sh @@ -0,0 +1,141 @@ +#!/bin/sh + +set -eu + +if [ $# -ne 2 ] ; then + cat <&2 +Usage: $(basename $0) sed-command output-file" + +Examples: +$(basename $0) 'target/release/sedapp sed' rust-sed +$(basename $0) /usr/bin/sed gnu-sed +EOF + exit 1 +fi + +PROG="$1" +OUT="$2" + +SCRIPTS=tests/fixtures/sed/script + + +# Run hyperfine with the specified name and command and collect the results. +bench_run() +{ + if hyperfine --command-name "$1" --warmup 2 --export-csv out.csv "$2" ; then + # Output the results sans-heading. + sed 1d out.csv >>"$OUT" + else + # Unable to run; output a named empty record. + echo "$1,,,,,,," >>"$OUT" + fi + + rm out.csv +} + +# Shared heading +echo 'command,mean,stddev,median,user,system,min,max' >"$OUT" + +# Short line processing +awk 'BEGIN { for (i = 0; i < 50000000; i++) { print i } }' > lines.txt + +# No operation +bench_run no-op-short "$PROG '' lines.txt" + +# Log file processing + +# Create an access.log file with the specified number of lines +create_access_log() +{ + awk 'BEGIN { + for (i = 0; i < 5000000; i++) { + printf("192.168.%d.%d - - [01/Jan/2024:00:00:00 +0000] \"GET /index.html HTTP/1.1\" 200 1234 \"-\" \"Mozilla/5.0\"\n", int(i/256)%256, i%256); + } + }' > access.log +} + +# Create long file for simple commands +create_access_log 5000000 + +# No operation +bench_run access-log-no-op "$PROG '' access.log" + +# No substitution +bench_run access-log-no-subst "$PROG s/Chrome/Chromium/ access.log" + +# Substitution +bench_run access-log-subst "$PROG s/Mozilla/Chromium/ access.log" + +# No deletion +bench_run access-log-no-del "$PROG /Chrome/d access.log" + +# Full deletion +bench_run access-log-all-del "$PROG /Mozilla/d access.log" + +# Create shorter file for more complex commands +create_access_log 50000 + +# Transliteration +bench_run access-log-translit "$PROG y/0123456789/9876543210/ access.log" + +# Multiple substitutions +bench_run access-log-complex-sub "$PROG -f $SCRIPTS/http-log-redact.sed access.log" + +rm access.log + +# Remove \r +awk 'BEGIN { + for (i = 0; i < 1500000; i++) { + printf("line %d with windows endings\r\n", i); + } +}' > legacy_input.txt + +bench_run remove-cr "$PROG 's/\r$//' legacy_input.txt" + +rm legacy_input.txt + +# Genomic data cleanup + +awk 'BEGIN { + for (i = 0; i < 1000000; i++) { + chr = "chr" (1 + i % 22); + printf("%s\t%d\t%s\t.\t%s\t.\t.\n", chr, i * 100, (i % 2 ? "A" : "T"), (i % 2 ? "G" : "C")); + } +}' > genome.tsv + +CMD='/^#/d; s/\t\./\tNA/g; s/\.$/NA/' +bench_run genome-subst "$PROG '$CMD' genome.tsv" + +rm -f genome.tsv + +# Number fixups: remove thousands separator, change , into . +awk 'BEGIN { + for (i = 0; i < 700000; i++) { + euros = int(i % 10000); + cents = int(i % 100); + thousands = int(euros / 1000); + remainder = euros % 1000; + printf("%d.%03d,%02d\n", thousands, remainder, cents); + } +}' > finance.csv + +CMD='s/\([0-9]\)\.\([0-9]\)/\1\2/g;s/\([0-9]\),\([0-9]\)/\1.\2/g' +bench_run number-fix "$PROG '$CMD' finance.csv" + +rm -f finance.csv + +# Long script compilation +for i in $(seq 1 99) ; do + awk -v tag="$i" '{ gsub(/[tb:] [[:alnum:]_]+/, "&" tag); print }' $SCRIPTS/math.sed +done >long_script.sed + +bench_run long-script "echo -n '' | $PROG -f long_script.sed " + +rm long_script.sed + + +# Towers of Hanoi +bench_run hanoi "echo ':abcdefghijkl: : :' | $PROG -f $SCRIPTS/hanoi.sed" + +# Arbitrary precision arithmetic +bench_run factorial "echo 30\! | $PROG -f $SCRIPTS/math.sed"