uutils/sed
0
0
Fork 0
mirror of https://github.com/uutils/sed.git synced 2026-06-10 16:14:15 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'optimize' into aci-commands

Browse Source
This commit is contained in:
Diomidis Spinellis
2025-05-28 15:39:44 +03:00
parent a233619686 5af9cf66b3
commit 27bdec6409
13 changed files with 1222 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+4
View File
@@ -703,7 +703,9 @@ dependencies = [
"ctor",
"fancy-regex",
"libc",
"memchr",
"memmap2",
"once_cell",
"phf",
"phf_codegen",
"pretty_assertions",
@@ -896,7 +898,9 @@ version = "0.0.1"
dependencies = [
"clap",
"fancy-regex",
"memchr",
"memmap2",
"once_cell",
"regex",
"tempfile",
"uucore",
Cargo.toml
+4
View File
@@ -38,7 +38,9 @@ clap_complete = "4.5"
clap_mangen = "0.2"
fancy-regex = "0.14.0"
libc = "0.2.153"
memchr = "2.7.4"
memmap2 = "0.9"
once_cell = "1.21.3"
phf = "0.11.2"
phf_codegen = "0.11.2"
rand = { version = "0.9", features = ["small_rng"] }
@@ -56,7 +58,9 @@ clap_complete = { workspace = true }
clap_mangen = { workspace = true }
ctor = "0.4.1"
fancy-regex = { workspace = true }
memchr = { workspace = true }
memmap2.workspace = true
once_cell = { workspace = true }
phf = { workspace = true }
sed = { optional = true, version = "0.0.1", package = "uu_sed", path = "src/uu/sed" }
sysinfo = { workspace = true }
README.md
+1
View File
@@ -42,6 +42,7 @@ cargo run --release
* In addition to `\n`, other escape sequences (octal, hex, C) are supported
in the strings of the `y` command.
Under POSIX these yield undefined behavior.
* The substitution command replacement group `\0` is a synonym for &.
### Supported BSD and GNU extensions
* The second address in a range can be specified as a relative address with +N.
src/uu/sed/Cargo.toml
+2
View File
@@ -15,9 +15,11 @@ categories = ["command-line-utilities"]
[dependencies]
clap = { workspace = true }
fancy-regex = { workspace = true }
memchr = { workspace = true }
regex = { workspace = true }
tempfile = { workspace = true }
memmap2 = { workspace = true }
once_cell = { workspace = true }
uucore = { workspace = true }
[lib]
src/uu/sed/src/command.rs
+119 -76
View File
@@ -11,15 +11,15 @@
// TODO: remove when compile is implemented
#![allow(dead_code)]
use crate::fast_regex::{Captures, Match, Regex};
use crate::named_writer::NamedWriter;
use fancy_regex::{Captures, Regex};
use std::borrow::Cow;
use std::cell::RefCell;
use std::collections::HashMap;
use std::path::PathBuf; // For file descriptors and equivalent
use std::rc::Rc;
use uucore::error::UResult;
use uucore::error::{UResult, USimpleError};
#[derive(Debug, Default, Clone)]
/// Compilation and processing options provided mostly through the
@@ -124,38 +124,65 @@ pub enum ReplacementPart {
/// All specified replacements for an RE
pub struct ReplacementTemplate {
pub parts: Vec<ReplacementPart>,
pub max_group_number: usize, // Highest used group number (e.g. 8 for \8)
}
impl Default for ReplacementTemplate {
/// Create an empty template.
fn default() -> Self {
ReplacementTemplate { parts: Vec::new() }
ReplacementTemplate::new(Vec::new())
}
}
impl ReplacementTemplate {
/// Construct from the parts
pub fn new(parts: Vec<ReplacementPart>) -> Self {
let max_group_number = parts
.iter()
.filter_map(|part| match part {
ReplacementPart::Group(n) => Some(*n),
_ => None,
})
.max()
.unwrap_or(0);
Self {
parts,
max_group_number: max_group_number.try_into().unwrap(),
}
}
/// Apply the template to the given RE captures.
/// Example:
/// let result = regex.replace_all(input, |caps: &Captures| {
/// template.apply(caps) });
/// template.apply_captures(caps) });
/// Returns an error if a backreference in the template was not matched by the RE.
pub fn apply(&self, caps: &Captures) -> UResult<String> {
pub fn apply_captures(&self, caps: &Captures) -> UResult<String> {
let mut result = String::new();
// Invalid group numbers may end here through (unkown at compile time)
// reused REs.
if self.max_group_number > caps.len() - 1 {
return Err(USimpleError::new(
2,
format!(
"invalid reference \\{} on `s' command's RHS",
self.max_group_number
),
));
}
for part in &self.parts {
match part {
ReplacementPart::Literal(s) => result.push_str(s),
ReplacementPart::WholeMatch => {
result.push_str(caps.get(0).map_or("", |m| m.as_str()));
result.push_str(caps.get(0)?.map(|m| m.as_str()).unwrap_or(""));
}
ReplacementPart::Group(n) => {
// Compilation guarantees we only get valid group numbers
result.push_str(
caps.get((*n).try_into().unwrap())
.map_or("", |m| m.as_str()),
);
let i: usize = (*n).try_into().unwrap();
result.push_str(caps.get(i)?.map(|m| m.as_str()).unwrap_or(""));
}
}
}
@@ -163,19 +190,22 @@ impl ReplacementTemplate {
Ok(result)
}
/// Returns the highest capture group number referenced in this template.
pub fn max_group_number(&self) -> u32 {
self.parts
.iter()
.filter_map(|part| {
if let ReplacementPart::Group(n) = part {
Some(*n)
} else {
None
/// Apply the template to the given RE single match.
pub fn apply_match(&self, m: &Match) -> String {
let mut result = String::new();
for part in &self.parts {
match part {
ReplacementPart::Literal(s) => result.push_str(s),
ReplacementPart::WholeMatch => result.push_str(m.as_str()),
ReplacementPart::Group(_) => {
panic!("unexpected Regex group replacement")
}
})
.max()
.unwrap_or(0)
}
}
result
}
}
@@ -322,12 +352,13 @@ pub struct InputAction {
#[cfg(test)]
mod tests {
use super::*;
use crate::fast_io::IOChunk;
// Return the captures for the RE applied to the specified string
fn caps_for<'a>(re: &str, input: &'a str) -> Captures<'a> {
fn caps_for<'a>(re: &str, chunk: &'a mut IOChunk) -> Captures<'a> {
Regex::new(re)
.unwrap()
.captures(input)
.captures(chunk)
.unwrap()
.expect("captures")
}
@@ -336,96 +367,108 @@ mod tests {
// s/foo//
fn test_empty_template() {
let template = ReplacementTemplate::default();
let caps = caps_for("foo", "foo");
let input = &mut IOChunk::from_str("foo");
let caps = caps_for("foo", input);
let result = template.apply(&caps).unwrap();
let result = template.apply_captures(&caps).unwrap();
assert_eq!(result, "");
}
#[test]
// s/abc/hello/
fn test_literal_only() {
let template = ReplacementTemplate {
parts: vec![ReplacementPart::Literal("hello".into())],
};
let caps = caps_for("abc", "abc");
let template = ReplacementTemplate::new(vec![ReplacementPart::Literal("hello".into())]);
let input = &mut IOChunk::from_str("abc");
let caps = caps_for("abc", input);
let result = template.apply(&caps).unwrap();
let result = template.apply_captures(&caps).unwrap();
assert_eq!(result, "hello");
}
#[test]
// s/foo\d+/got: &/
fn test_whole_match() {
let template = ReplacementTemplate {
parts: vec![
ReplacementPart::Literal("got: ".into()),
ReplacementPart::WholeMatch,
],
};
let caps = caps_for(r"foo\d+", "foo42");
let template = ReplacementTemplate::new(vec![
ReplacementPart::Literal("got: ".into()),
ReplacementPart::WholeMatch,
]);
let input = &mut IOChunk::from_str("foo42");
let caps = caps_for(r"foo\d+", input);
let result = template.apply(&caps).unwrap();
let result = template.apply_captures(&caps).unwrap();
assert_eq!(result, "got: foo42");
}
#[test]
// s/foo(\d+)/number: \1/
fn test_backreference() {
let template = ReplacementTemplate {
parts: vec![
ReplacementPart::Literal("number: ".into()),
ReplacementPart::Group(1),
],
};
let caps = caps_for(r"foo(\d+)", "foo42");
let template = ReplacementTemplate::new(vec![
ReplacementPart::Literal("number: ".into()),
ReplacementPart::Group(1),
]);
let input = &mut IOChunk::from_str("foo42");
let caps = caps_for(r"foo(\d+)", input);
let result = template.apply(&caps).unwrap();
let result = template.apply_captures(&caps).unwrap();
assert_eq!(result, "number: 42");
}
#[test]
// s/(\w+):(\d+)/key: \1, value: \2/
fn test_multiple_parts() {
let template = ReplacementTemplate {
parts: vec![
ReplacementPart::Literal("key: ".into()),
ReplacementPart::Group(1),
ReplacementPart::Literal(", value: ".into()),
ReplacementPart::Group(2),
],
};
let caps = caps_for(r"(\w+):(\d+)", "x:123");
let template = ReplacementTemplate::new(vec![
ReplacementPart::Literal("key: ".into()),
ReplacementPart::Group(1),
ReplacementPart::Literal(", value: ".into()),
ReplacementPart::Group(2),
]);
let input = &mut IOChunk::from_str("x:123");
let caps = caps_for(r"(\w+):(\d+)", input);
let result = template.apply(&caps).unwrap();
let result = template.apply_captures(&caps).unwrap();
assert_eq!(result, "key: x, value: 123");
}
#[test]
// s/(\w+):(\d+)/key: \1, value: \3/
fn test_invalid_group() {
let template = ReplacementTemplate::new(vec![
ReplacementPart::Literal("key: ".into()),
ReplacementPart::Group(1),
ReplacementPart::Literal(", value: ".into()),
ReplacementPart::Group(3),
]);
let input = &mut IOChunk::from_str("x:123");
let caps = caps_for(r"(\w+):(\d+)", input);
let result = template.apply_captures(&caps);
assert!(result.is_err());
let msg = result.unwrap_err().to_string();
assert!(msg.contains("invalid reference \\3"));
}
// max_group_number
#[test]
fn test_max_group_number_with_groups() {
let template = ReplacementTemplate {
parts: vec![
ReplacementPart::Literal("a".into()),
ReplacementPart::Group(2),
ReplacementPart::WholeMatch,
ReplacementPart::Group(5),
ReplacementPart::Literal("z".into()),
],
};
assert_eq!(template.max_group_number(), 5);
let template = ReplacementTemplate::new(vec![
ReplacementPart::Literal("a".into()),
ReplacementPart::Group(2),
ReplacementPart::WholeMatch,
ReplacementPart::Group(5),
ReplacementPart::Literal("z".into()),
]);
assert_eq!(template.max_group_number, 5);
}
#[test]
fn test_max_group_number_without_groups() {
let template = ReplacementTemplate {
parts: vec![
ReplacementPart::Literal("no".into()),
ReplacementPart::WholeMatch,
ReplacementPart::Literal("groups".into()),
],
};
assert_eq!(template.max_group_number(), 0);
let template = ReplacementTemplate::new(vec![
ReplacementPart::Literal("no".into()),
ReplacementPart::WholeMatch,
ReplacementPart::Literal("groups".into()),
]);
assert_eq!(template.max_group_number, 0);
}
// Transliteration
src/uu/sed/src/compiler.rs
+64 -28
View File
@@ -15,10 +15,10 @@ use crate::command::{
use crate::delimited_parser::{
compilation_error, parse_char_escape, parse_regex, parse_transliteration,
};
use crate::fast_regex::Regex;
use crate::named_writer::NamedWriter;
use crate::script_char_provider::ScriptCharProvider;
use crate::script_line_provider::ScriptLineProvider;
use fancy_regex::Regex;
use std::borrow::Cow;
use std::cell::RefCell;
use std::collections::HashMap;
@@ -706,14 +706,18 @@ pub fn compile_replacement(
}
match line.current() {
// \1 - \9
c @ '1'..='9' => {
// \0 - \9
c @ '0'..='9' => {
let ref_num = c.to_digit(10).unwrap();
if !literal.is_empty() {
parts.push(ReplacementPart::Literal(std::mem::take(&mut literal)));
}
parts.push(ReplacementPart::Group(ref_num));
if ref_num == 0 {
parts.push(ReplacementPart::WholeMatch);
} else {
parts.push(ReplacementPart::Group(ref_num));
}
line.advance();
}
@@ -762,7 +766,7 @@ pub fn compile_replacement(
if !literal.is_empty() {
parts.push(ReplacementPart::Literal(literal));
}
return Ok(ReplacementTemplate { parts });
return Ok(ReplacementTemplate::new(parts));
}
c => {
@@ -816,11 +820,21 @@ fn compile_subst_command(
);
}
if pattern.is_empty() {
subst.regex = None; // Will be reuse saved RE at runtime.
} else {
// Compile regex with now known ignore_case flag.
subst.regex = compile_regex(lines, line, &pattern, context, subst.ignore_case)?;
// Compile regex with now known ignore_case flag.
subst.regex = compile_regex(lines, line, &pattern, context, subst.ignore_case)?;
// Catch invalid group references at compile time, if possible.
if let Some(regex) = &subst.regex {
if subst.replacement.max_group_number > regex.captures_len() - 1 {
return compilation_error(
lines,
line,
format!(
"invalid reference \\{} on `s' command's RHS",
subst.replacement.max_group_number
),
);
}
}
cmd.data = CommandData::Substitution(subst);
@@ -1153,6 +1167,7 @@ fn lookup_command(cmd: char) -> Option<&'static CommandSpec> {
#[cfg(test)]
mod tests {
use super::*;
use crate::fast_io::IOChunk;
// Return an empty line provider and a char provider for the specified str.
fn make_providers(input: &str) -> (ScriptLineProvider, ScriptCharProvider) {
@@ -1370,8 +1385,8 @@ mod tests {
let regex = compile_regex(&lines, &chars, "abc", &ctx(), false)
.unwrap()
.expect("regex should be present");
assert!(regex.is_match("abc").unwrap());
assert!(!regex.is_match("ABC").unwrap());
assert!(regex.is_match(&mut IOChunk::from_str("abc")).unwrap());
assert!(!regex.is_match(&mut IOChunk::from_str("ABC")).unwrap());
}
#[test]
@@ -1380,9 +1395,9 @@ mod tests {
let regex = compile_regex(&lines, &chars, "abc", &ctx(), true)
.unwrap()
.expect("regex should be present");
assert!(regex.is_match("abc").unwrap());
assert!(regex.is_match("ABC").unwrap());
assert!(regex.is_match("AbC").unwrap());
assert!(regex.is_match(&mut IOChunk::from_str("abc")).unwrap());
assert!(regex.is_match(&mut IOChunk::from_str("ABC")).unwrap());
assert!(regex.is_match(&mut IOChunk::from_str("AbC")).unwrap());
}
#[test]
@@ -1430,7 +1445,7 @@ mod tests {
let addr = compile_address(&lines, &mut chars, &ctx()).unwrap();
assert!(matches!(addr.atype, AddressType::Re));
if let AddressValue::Regex(Some(re)) = addr.value {
assert!(re.is_match("hello").unwrap());
assert!(re.is_match(&mut IOChunk::from_str("hello")).unwrap());
} else {
panic!("expected Regex address value");
}
@@ -1442,7 +1457,7 @@ mod tests {
let addr = compile_address(&lines, &mut chars, &ctx()).unwrap();
assert!(matches!(addr.atype, AddressType::Re));
if let AddressValue::Regex(Some(re)) = addr.value {
assert!(re.is_match("hello").unwrap());
assert!(re.is_match(&mut IOChunk::from_str("hello")).unwrap());
} else {
panic!("expected Regex address value");
}
@@ -1454,7 +1469,7 @@ mod tests {
let addr = compile_address(&lines, &mut chars, &ctx()).unwrap();
assert!(matches!(addr.atype, AddressType::Re));
if let AddressValue::Regex(Some(re)) = addr.value {
assert!(!re.is_match("helio").unwrap());
assert!(!re.is_match(&mut IOChunk::from_str("helio")).unwrap());
} else {
panic!("expected Regex address value");
}
@@ -1466,7 +1481,7 @@ mod tests {
let addr = compile_address(&lines, &mut chars, &ctx()).unwrap();
assert!(matches!(addr.atype, AddressType::Re));
if let AddressValue::Regex(Some(re)) = addr.value {
assert!(re.is_match("hello").unwrap());
assert!(re.is_match(&mut IOChunk::from_str("hello")).unwrap());
} else {
panic!("expected Regex address value");
}
@@ -1478,7 +1493,7 @@ mod tests {
let addr = compile_address(&lines, &mut chars, &ctx()).unwrap();
assert!(matches!(addr.atype, AddressType::Re));
if let AddressValue::Regex(Some(re)) = addr.value {
assert!(re.is_match("HELLO").unwrap()); // case-insensitive
assert!(re.is_match(&mut IOChunk::from_str("HELLO")).unwrap()); // case-insensitive
} else {
panic!("expected Regex address value");
}
@@ -1569,8 +1584,8 @@ mod tests {
AddressType::Re
));
if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr1.as_ref().unwrap().value {
assert!(re.is_match("foo").unwrap());
assert!(!re.is_match("bar").unwrap());
assert!(re.is_match(&mut IOChunk::from_str("foo")).unwrap());
assert!(!re.is_match(&mut IOChunk::from_str("bar")).unwrap());
} else {
panic!("expected a regex address");
};
@@ -1589,8 +1604,8 @@ mod tests {
AddressType::Re
));
if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr1.as_ref().unwrap().value {
assert!(re.is_match("foo").unwrap());
assert!(!re.is_match("bar").unwrap());
assert!(re.is_match(&mut IOChunk::from_str("foo")).unwrap());
assert!(!re.is_match(&mut IOChunk::from_str("bar")).unwrap());
} else {
panic!("expected a regex address");
}
@@ -1600,8 +1615,8 @@ mod tests {
AddressType::Re
));
if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr2.as_ref().unwrap().value {
assert!(re.is_match("bar").unwrap());
assert!(!re.is_match("foo").unwrap());
assert!(re.is_match(&mut IOChunk::from_str("bar")).unwrap());
assert!(!re.is_match(&mut IOChunk::from_str("foo")).unwrap());
} else {
panic!("expected a regex address");
};
@@ -1619,8 +1634,8 @@ mod tests {
AddressType::Re
));
if let AddressValue::Regex(Some(re)) = &cmd.borrow().addr1.as_ref().unwrap().value {
assert!(re.is_match("FOO").unwrap());
assert!(re.is_match("foo").unwrap());
assert!(re.is_match(&mut IOChunk::from_str("FOO")).unwrap());
assert!(re.is_match(&mut IOChunk::from_str("foo")).unwrap());
} else {
panic!("expected a regex address with case-insensitive match");
};
@@ -1796,6 +1811,18 @@ mod tests {
assert!(matches!(&template.parts[1], ReplacementPart::WholeMatch));
}
#[test]
fn test_compile_replacement_whole_match_synonym() {
let (mut lines, mut chars) = make_providers(r"/The match was: \0/");
let template = compile_replacement(&mut lines, &mut chars).unwrap();
assert_eq!(template.parts.len(), 2);
assert!(
matches!(&template.parts[0], ReplacementPart::Literal(s) if s == "The match was: ")
);
assert!(matches!(&template.parts[1], ReplacementPart::WholeMatch));
}
#[test]
fn test_compile_replacement_ampersand() {
let (mut lines, mut chars) = make_providers("/Simon \\& Garfunkel/");
@@ -1989,6 +2016,15 @@ mod tests {
}
}
#[test]
fn test_compile_subst_invalid_group_reference() {
let (mut lines, mut chars) = make_providers(r"s/f(o)o/\2/");
let mut cmd = Command::default();
let err = compile_subst_command(&mut lines, &mut chars, &mut cmd, &ctx()).unwrap_err();
assert!(err.to_string().contains("invalid reference \\2"));
}
// bre_to_ere
#[test]
fn test_bre_group_translation() {
src/uu/sed/src/fast_io.rs
+42 -18
View File
@@ -14,9 +14,12 @@
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.
#[cfg(unix)]
use memchr::memchr;
#[cfg(unix)]
use memmap2::Mmap;
use std::cell::Cell;
use std::fs::File;
use std::io::{self, BufRead, BufReader, BufWriter, Read, Write};
@@ -69,10 +72,12 @@ impl<'a> MmapLineCursor<'a> {
}
let start = self.pos;
let mut end = start;
while end < self.data.len() && self.data[end] != b'\n' {
end += 1;
}
let mut end = if let Some(pos) = memchr(b'\n', &self.data[start..]) {
pos + start
} else {
self.data.len()
};
if end < self.data.len() {
end += 1; // include \n in full span
@@ -135,7 +140,7 @@ impl ReadLineCursor {
/// As chunk of data that is input and can be output, often very efficiently
#[derive(Debug, PartialEq, Eq)]
pub struct IOChunk<'a> {
utf8_verified: bool, // True if the contents are valid UTF-8
utf8_verified: Cell<bool>, // True if the contents are valid UTF-8
content: IOChunkContent<'a>,
}
@@ -143,14 +148,14 @@ impl<'a> IOChunk<'a> {
/// Construct an IOChunk from the given content
fn from_content(content: IOChunkContent<'a>) -> Self {
Self {
utf8_verified: false,
utf8_verified: Cell::new(false),
content,
}
}
/// Clear the object's contents, converting it into Owned if needed.
pub fn clear(&mut self) {
self.utf8_verified = true;
self.utf8_verified.set(true);
match &mut self.content {
IOChunkContent::Owned {
content,
@@ -182,10 +187,19 @@ impl<'a> IOChunk<'a> {
}
}
#[cfg(test)]
/// Create an Owned newline-terminated IOChunk from a string.
pub fn from_str(s: &str) -> Self {
IOChunk {
content: IOChunkContent::new_owned(s.to_string(), true),
utf8_verified: Cell::new(false),
}
}
/// Set the object's contents to the specified string.
/// Convert it into Owned if needed.
pub fn set_to_string(&mut self, new_content: String, add_newline: bool) {
self.utf8_verified = true;
self.utf8_verified.set(true);
match &mut self.content {
IOChunkContent::Owned {
content,
@@ -203,16 +217,16 @@ impl<'a> IOChunk<'a> {
}
/// Return the content as a str.
pub fn as_str(&mut self) -> Result<&str, Box<dyn UError>> {
pub fn as_str(&self) -> Result<&str, Box<dyn UError>> {
match &self.content {
#[cfg(unix)]
IOChunkContent::MmapInput { content, .. } => {
if self.utf8_verified {
if self.utf8_verified.get() {
// Use cached result
Ok(unsafe { self.content.as_str_unchecked() })
} else {
let result = str::from_utf8(content);
self.utf8_verified = true;
self.utf8_verified.set(true);
result.map_err(|e| USimpleError::new(1, e.to_string()))
}
}
@@ -220,6 +234,15 @@ impl<'a> IOChunk<'a> {
}
}
/// Return the raw byte content (always safe).
pub fn as_bytes(&self) -> &[u8] {
match &self.content {
#[cfg(unix)]
IOChunkContent::MmapInput { content, .. } => content,
IOChunkContent::Owned { content, .. } => content.as_bytes(),
}
}
/// Convert content to the Owned variant if it's not already.
/// Fails if the conversion to UTF-8 fails.
pub fn ensure_owned(&mut self) -> Result<(), Box<dyn UError>> {
@@ -232,7 +255,7 @@ impl<'a> IOChunk<'a> {
let has_newline = full_span.last().copied() == Some(b'\n');
self.content =
IOChunkContent::new_owned(valid_str.to_string(), has_newline);
self.utf8_verified = true;
self.utf8_verified.set(true);
Ok(())
}
Err(e) => Err(USimpleError::new(1, e.to_string())),
@@ -934,7 +957,7 @@ mod tests {
{
assert_eq!(content, "first line");
assert!(has_newline);
assert!(!utf8_verified);
assert!(!utf8_verified.get());
assert!(!last_line);
} else {
panic!("Expected IOChunkContent::Owned");
@@ -960,7 +983,7 @@ mod tests {
panic!("Expected IOChunkContent::Owned");
}
if let Some((mut content, last_line)) = reader.get_line()? {
if let Some((content, last_line)) = reader.get_line()? {
assert_eq!(content.as_str().unwrap(), "last line");
assert!(last_line);
} else {
@@ -998,7 +1021,7 @@ mod tests {
{
assert_eq!(content, b"first line");
assert_eq!(full_span, b"first line\n");
assert!(!utf8_verified);
assert!(!utf8_verified.get());
assert!(!last_line);
} else {
panic!("Expected IOChunkContent::MapInput");
@@ -1018,15 +1041,16 @@ mod tests {
{
assert_eq!(content, b"second line");
assert_eq!(full_span, b"second line\n");
assert!(!utf8_verified);
assert!(!utf8_verified.get());
assert!(!last_line);
} else {
panic!("Expected IOChunkContent::MapInput");
}
if let Some((mut content, last_line)) = reader.get_line()? {
if let Some((content, last_line)) = reader.get_line()? {
assert_eq!(content.as_bytes(), b"last line");
assert_eq!(content.as_str().unwrap(), "last line");
assert!(content.utf8_verified);
assert!(content.utf8_verified.get());
assert!(last_line);
// Cached version
assert_eq!(content.as_str().unwrap(), "last line");
src/uu/sed/src/fast_regex.rs
+745
View File
File diff suppressed because it is too large Load Diff
src/uu/sed/src/processor.rs
+65 -28
View File
@@ -13,9 +13,10 @@ use crate::command::{
ProcessingContext, Substitution, Transliteration,
};
use crate::fast_io::{IOChunk, LineReader, OutputBuffer};
use crate::fast_regex::Regex;
use crate::in_place::InPlace;
use crate::named_writer;
use fancy_regex::Regex;
use std::cell::RefCell;
use std::io::{self, IsTerminal};
use std::path::PathBuf;
@@ -32,10 +33,7 @@ fn match_address(
AddressType::Re => {
if let AddressValue::Regex(ref re) = addr.value {
let regex = re_or_saved_re(re, context)?;
let matched = regex.is_match(pattern.as_str()?).map_err(|e| {
USimpleError::new(2, format!("regular expression match error: {}", e))
})?;
Ok(matched)
regex.is_match(pattern)
} else {
Ok(false)
}
@@ -197,39 +195,78 @@ fn substitute(
let mut result = String::new();
let mut replaced = false;
let text = pattern.as_str()?;
let mut text: Option<&str> = None;
let regex = re_or_saved_re(&sub.regex, context)?;
for caps in regex.captures_iter(text) {
count += 1;
let caps = caps.map_err(|e| {
USimpleError::new(
2,
format!("regular expression capture retrieval error: {}", e),
)
})?;
match (sub.occurrence, sub.replacement.max_group_number) {
(1, 0) => {
// Example: s/foo/bar/: find() is enough.
let m = regex.find(pattern)?;
if let Some(m) = m {
text = Some(pattern.as_str()?);
result.push_str(&text.unwrap()[last_end..m.start()]);
let m = caps.get(0).unwrap();
// Always write the unmatched text before this match.
result.push_str(&text[last_end..m.start()]);
if sub.occurrence == 0 || count == sub.occurrence {
let replacement = sub.replacement.apply(&caps)?;
result.push_str(&replacement);
replaced = true;
} else {
// Not the target match — leave the match unchanged.
result.push_str(m.as_str());
let replacement = sub.replacement.apply_match(&m);
result.push_str(&replacement);
replaced = true;
last_end = m.end();
}
}
last_end = m.end();
(1, _) => {
// Example: s/\(.\)\(.\)/\2\1/: captures() is enough.
let caps = regex.captures(pattern)?;
if let Some(caps) = caps {
let m = caps.get(0)?.unwrap();
text = Some(pattern.as_str()?);
result.push_str(&text.unwrap()[last_end..m.start()]);
let replacement = sub.replacement.apply_captures(&caps)?;
result.push_str(&replacement);
replaced = true;
last_end = m.end();
}
}
(_, _) => {
// Example: s/(.)(.)/\2\1/3: captures_iter() is needed.
// Iterate over multiple captures of the RE in the pattern.
for caps in regex.captures_iter(pattern)? {
count += 1;
let caps = caps?;
let m = caps.get(0)?.unwrap();
// Always write the unmatched text before this match.
if text.is_none() {
text = Some(pattern.as_str()?);
}
result.push_str(&text.unwrap()[last_end..m.start()]);
if sub.occurrence == 0 || count == sub.occurrence {
let replacement = sub.replacement.apply_captures(&caps)?;
result.push_str(&replacement);
replaced = true;
} else {
// Not the target match — leave the match unchanged.
result.push_str(m.as_str());
}
last_end = m.end();
// Early exit if only a specific occurrence,
// (likely 1) needed replacing.
if count == sub.occurrence {
break;
}
}
}
}
// Handle substitution success.
if replaced {
result.push_str(&text[last_end..]);
result.push_str(&text.unwrap()[last_end..]);
pattern.set_to_string(result, pattern.is_newline_terminated());
src/uu/sed/src/sed.rs
+1
View File
@@ -12,6 +12,7 @@ pub mod command;
pub mod compiler;
pub mod delimited_parser;
pub mod fast_io;
pub mod fast_regex;
pub mod in_place;
pub mod named_writer;
pub mod processor;
tests/fixtures/sed/script/http-log-redact.sed
Vendored
+4
View File
@@ -0,0 +1,4 @@
#!/
s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[REDACTED_IP]/;
s/\[[0-9]{2}\/[A-Za-z]{3}\/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} [+-][0-9]{4}\]/[TIMESTAMP]/;
s/"Mozilla[^"]*"/"[UA]"/
util/bench-compare.sh
Executable
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
#
# Compare the results of two benchmarks reporting fastest for each task
#
set -eu
if [ $# -ne 2 ] ; then
echo "Usage: $(basename $0) results-1 results-2" 1>&2
exit 1
fi
paste -d , "$1" "$2" |
sed "s/^/$1,$2,/" |
awk -F, 'FNR != 1 {
printf("%22s ", $3);
if (!$4)
printf("%12s encountered an error\n", $1);
else if (!$12)
printf("%12s encountered an error\n", $2);
else if ($4 / $12 > 1.01)
printf("%12s is %5.2f times faster than %s\n", $2, $4 / $12, $1);
else if ($12 / $4 > 1.01)
printf("%12s is %5.2f times faster than %s\n", $1, $12 / $4, $2);
else
printf("%12s is similarly fast as %s\n", $1, $2);
}'
util/benchmark.sh
Executable
+141
View File
@@ -0,0 +1,141 @@
#!/bin/sh
set -eu
if [ $# -ne 2 ] ; then
cat <<EOF 1>&2
Usage: $(basename $0) sed-command output-file"
Examples:
$(basename $0) 'target/release/sedapp sed' rust-sed
$(basename $0) /usr/bin/sed gnu-sed
EOF
exit 1
fi
PROG="$1"
OUT="$2"
SCRIPTS=tests/fixtures/sed/script
# Run hyperfine with the specified name and command and collect the results.
bench_run()
{
if hyperfine --command-name "$1" --warmup 2 --export-csv out.csv "$2" ; then
# Output the results sans-heading.
sed 1d out.csv >>"$OUT"
else
# Unable to run; output a named empty record.
echo "$1,,,,,,," >>"$OUT"
fi
rm out.csv
}
# Shared heading
echo 'command,mean,stddev,median,user,system,min,max' >"$OUT"
# Short line processing
awk 'BEGIN { for (i = 0; i < 50000000; i++) { print i } }' > lines.txt
# No operation
bench_run no-op-short "$PROG '' lines.txt"
# Log file processing
# Create an access.log file with the specified number of lines
create_access_log()
{
awk 'BEGIN {
for (i = 0; i < 5000000; i++) {
printf("192.168.%d.%d - - [01/Jan/2024:00:00:00 +0000] \"GET /index.html HTTP/1.1\" 200 1234 \"-\" \"Mozilla/5.0\"\n", int(i/256)%256, i%256);
}
}' > access.log
}
# Create long file for simple commands
create_access_log 5000000
# No operation
bench_run access-log-no-op "$PROG '' access.log"
# No substitution
bench_run access-log-no-subst "$PROG s/Chrome/Chromium/ access.log"
# Substitution
bench_run access-log-subst "$PROG s/Mozilla/Chromium/ access.log"
# No deletion
bench_run access-log-no-del "$PROG /Chrome/d access.log"
# Full deletion
bench_run access-log-all-del "$PROG /Mozilla/d access.log"
# Create shorter file for more complex commands
create_access_log 50000
# Transliteration
bench_run access-log-translit "$PROG y/0123456789/9876543210/ access.log"
# Multiple substitutions
bench_run access-log-complex-sub "$PROG -f $SCRIPTS/http-log-redact.sed access.log"
rm access.log
# Remove \r
awk 'BEGIN {
for (i = 0; i < 1500000; i++) {
printf("line %d with windows endings\r\n", i);
}
}' > legacy_input.txt
bench_run remove-cr "$PROG 's/\r$//' legacy_input.txt"
rm legacy_input.txt
# Genomic data cleanup
awk 'BEGIN {
for (i = 0; i < 1000000; i++) {
chr = "chr" (1 + i % 22);
printf("%s\t%d\t%s\t.\t%s\t.\t.\n", chr, i * 100, (i % 2 ? "A" : "T"), (i % 2 ? "G" : "C"));
}
}' > genome.tsv
CMD='/^#/d; s/\t\./\tNA/g; s/\.$/NA/'
bench_run genome-subst "$PROG '$CMD' genome.tsv"
rm -f genome.tsv
# Number fixups: remove thousands separator, change , into .
awk 'BEGIN {
for (i = 0; i < 700000; i++) {
euros = int(i % 10000);
cents = int(i % 100);
thousands = int(euros / 1000);
remainder = euros % 1000;
printf("%d.%03d,%02d\n", thousands, remainder, cents);
}
}' > finance.csv
CMD='s/\([0-9]\)\.\([0-9]\)/\1\2/g;s/\([0-9]\),\([0-9]\)/\1.\2/g'
bench_run number-fix "$PROG '$CMD' finance.csv"
rm -f finance.csv
# Long script compilation
for i in $(seq 1 99) ; do
awk -v tag="$i" '{ gsub(/[tb:] [[:alnum:]_]+/, "&" tag); print }' $SCRIPTS/math.sed
done >long_script.sed
bench_run long-script "echo -n '' | $PROG -f long_script.sed "
rm long_script.sed
# Towers of Hanoi
bench_run hanoi "echo ':abcdefghijkl: : :' | $PROG -f $SCRIPTS/hanoi.sed"
# Arbitrary precision arithmetic
bench_run factorial "echo 30\! | $PROG -f $SCRIPTS/math.sed"