mirror of
https://github.com/uutils/grep.git
synced 2026-06-10 16:15:11 -07:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
be51c04c08 | ||
|
|
da32a63663 | ||
|
|
9c21a7d2f0 | ||
|
|
e767a30c1a | ||
|
|
1a3e8a391d |
@@ -1,50 +0,0 @@
|
||||
name: Benchmarks
|
||||
|
||||
# spell-checker:ignore codspeed dtolnay Swatinem sccache
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, master ]
|
||||
pull_request:
|
||||
branches: [ main, master ]
|
||||
|
||||
permissions:
|
||||
contents: read # to fetch code (actions/checkout)
|
||||
|
||||
# End the current execution if there is a new changeset in the PR.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
|
||||
|
||||
jobs:
|
||||
benchmarks:
|
||||
name: Run benchmarks (CodSpeed)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
|
||||
- name: Run sccache-cache
|
||||
uses: mozilla-actions/sccache-action@v0.0.10
|
||||
|
||||
- name: Install cargo-codspeed
|
||||
uses: taiki-e/install-action@v2
|
||||
with:
|
||||
tool: cargo-codspeed
|
||||
|
||||
- name: Build benchmarks
|
||||
run: cargo codspeed build -p uu_grep
|
||||
|
||||
- name: Run benchmarks
|
||||
uses: CodSpeedHQ/action@v4
|
||||
env:
|
||||
CODSPEED_LOG: debug
|
||||
with:
|
||||
mode: simulation
|
||||
run: cargo codspeed run -p uu_grep > /dev/null
|
||||
token: ${{ secrets.CODSPEED_TOKEN }}
|
||||
Generated
+11
-302
File diff suppressed because it is too large
Load Diff
@@ -28,11 +28,4 @@ uucore = "0.8.0"
|
||||
walkdir = "2.5"
|
||||
|
||||
[dev-dependencies]
|
||||
divan = { package = "codspeed-divan-compat", version = "4.0.5" }
|
||||
tempfile = "3.10.1"
|
||||
uucore = { version = "0.8.0", features = ["benchmark"] }
|
||||
uutests = "0.8.0"
|
||||
|
||||
[[bench]]
|
||||
name = "grep_bench"
|
||||
harness = false
|
||||
|
||||
@@ -33,7 +33,6 @@ cargo test
|
||||
|
||||
* Does not take `LANG`, etc., into account for handling file encodings (non-UTF8 matches are treated as binary)
|
||||
* No localization support yet
|
||||
* Performances need to be improved
|
||||
|
||||
## Contributing
|
||||
|
||||
|
||||
@@ -1,255 +0,0 @@
|
||||
// Benchmarks for the grep utility
|
||||
//
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// This file is part of the uutils grep package.
|
||||
// It is licensed under the MIT License.
|
||||
// For the full copyright and license information, please view the LICENSE
|
||||
// file that was distributed with this source code.
|
||||
|
||||
use divan::{Bencher, black_box};
|
||||
use uu_grep::uumain;
|
||||
use uucore::benchmark::{create_test_file, run_util_function};
|
||||
|
||||
/// Build an access-log-like data set with `n` lines.
|
||||
///
|
||||
/// Roughly a quarter of the lines use a non-default HTTP method / status /
|
||||
/// user-agent so that selective patterns match a realistic subset rather than
|
||||
/// every line or no line at all.
|
||||
fn access_log(n: usize) -> Vec<u8> {
|
||||
let mut data = Vec::new();
|
||||
for i in 0..n {
|
||||
let method = if i % 4 == 0 { "POST" } else { "GET" };
|
||||
let status = if i % 7 == 0 { 404 } else { 200 };
|
||||
let agent = if i % 3 == 0 {
|
||||
"Mozilla/5.0 (X11; Linux x86_64) Chrome/120.0"
|
||||
} else {
|
||||
"curl/8.5.0"
|
||||
};
|
||||
let line = format!(
|
||||
"192.168.{}.{} - - [01/Jan/2024:00:00:00 +0000] \"{} /index.html HTTP/1.1\" {} 1234 \"-\" \"{}\"\n",
|
||||
(i / 256) % 256,
|
||||
i % 256,
|
||||
method,
|
||||
status,
|
||||
agent,
|
||||
);
|
||||
data.extend_from_slice(line.as_bytes());
|
||||
}
|
||||
data
|
||||
}
|
||||
|
||||
/// Benchmark a literal search that matches nothing.
|
||||
///
|
||||
/// This is the purest measure of raw scan throughput: the whole file is read
|
||||
/// and searched but no output is produced.
|
||||
#[divan::bench]
|
||||
fn literal_no_match(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(
|
||||
uumain,
|
||||
&["ZZZ_NONEXISTENT_PATTERN_ZZZ", file_path_str],
|
||||
));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark a literal search that matches a subset of lines.
|
||||
#[divan::bench]
|
||||
fn literal_match_some(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(uumain, &["POST", file_path_str]));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark a literal search that matches every line (counting only).
|
||||
///
|
||||
/// `-c` keeps the output bounded so the benchmark measures matching rather than
|
||||
/// terminal I/O.
|
||||
#[divan::bench]
|
||||
fn literal_match_all_count(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(uumain, &["-c", "HTTP", file_path_str]));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark a fixed-string search (`-F`).
|
||||
#[divan::bench]
|
||||
fn fixed_string(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(
|
||||
uumain,
|
||||
&["-F", "Chrome/120.0", file_path_str],
|
||||
));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark a case-insensitive search (`-i`).
|
||||
#[divan::bench]
|
||||
fn case_insensitive(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(uumain, &["-i", "mozilla", file_path_str]));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark counting matches (`-c`).
|
||||
#[divan::bench]
|
||||
fn count(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(uumain, &["-c", "POST", file_path_str]));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark an inverted match (`-v`).
|
||||
///
|
||||
/// Most lines do not contain "POST", so this selects the majority of lines;
|
||||
/// `-c` bounds the output.
|
||||
#[divan::bench]
|
||||
fn invert_match_count(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(uumain, &["-vc", "POST", file_path_str]));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark printing line numbers (`-n`).
|
||||
#[divan::bench]
|
||||
fn line_number(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(1_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(uumain, &["-nc", "POST", file_path_str]));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark word-boundary matching (`-w`).
|
||||
#[divan::bench]
|
||||
fn word_match(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(uumain, &["-wc", "GET", file_path_str]));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark an extended regular expression with alternation (`-E`).
|
||||
#[divan::bench]
|
||||
fn extended_regex(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(
|
||||
uumain,
|
||||
&["-Ec", "(POST|DELETE|PUT)", file_path_str],
|
||||
));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark a basic regular expression with an anchor and character class.
|
||||
#[divan::bench]
|
||||
fn basic_regex(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(
|
||||
uumain,
|
||||
&["-c", "^192\\.168\\.[0-9]*\\.0 ", file_path_str],
|
||||
));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark a Perl-compatible regular expression (`-P`).
|
||||
#[divan::bench]
|
||||
fn perl_regex(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(
|
||||
uumain,
|
||||
&["-Pc", "\"\\d{3}\" \\d+", file_path_str],
|
||||
));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark `--only-matching` (`-o`) extracting a substring from each line.
|
||||
#[divan::bench]
|
||||
fn only_matching(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(1_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(
|
||||
uumain,
|
||||
&["-Eoc", "[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+", file_path_str],
|
||||
));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark quiet mode (`-q`), which can stop at the first match.
|
||||
#[divan::bench]
|
||||
fn quiet_first_match(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = create_test_file(&access_log(2_000_000), temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(uumain, &["-q", "POST", file_path_str]));
|
||||
});
|
||||
}
|
||||
|
||||
/// Benchmark searching short numeric lines (many small lines).
|
||||
#[divan::bench]
|
||||
fn short_lines(bencher: Bencher) {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let mut data = Vec::new();
|
||||
for i in 0..10_000_000 {
|
||||
data.extend_from_slice(format!("{i}\n").as_bytes());
|
||||
}
|
||||
let file_path = create_test_file(&data, temp_dir.path());
|
||||
let file_path_str = file_path.to_str().unwrap();
|
||||
|
||||
bencher.bench(|| {
|
||||
black_box(run_util_function(uumain, &["-c", "999", file_path_str]));
|
||||
});
|
||||
}
|
||||
|
||||
fn main() {
|
||||
divan::main();
|
||||
}
|
||||
+12
@@ -342,6 +342,13 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
|
||||
ColorMode::Never => false,
|
||||
ColorMode::Auto => std::io::stdout().is_terminal(),
|
||||
};
|
||||
// GREP_COLOR is deprecated in favour of GREP_COLORS' `mt` capability;
|
||||
// GNU warns about it, but only when color output is actually produced.
|
||||
if use_color && !grep_color.is_empty() {
|
||||
eprintln!(
|
||||
"grep: warning: GREP_COLOR='{grep_color}' is deprecated; use GREP_COLORS='mt={grep_color}'"
|
||||
);
|
||||
}
|
||||
let color_config = ColorConfig::from_env(&grep_color, &grep_colors);
|
||||
|
||||
let config = Config {
|
||||
@@ -903,6 +910,11 @@ impl<'a> ColorConfig<'a> {
|
||||
for item in grep_colors.split(':') {
|
||||
if let Some((key, value)) = item.split_once('=') {
|
||||
match key {
|
||||
// `mt` sets both the selected- and context-match colors.
|
||||
"mt" => {
|
||||
config.matched_selected = value;
|
||||
config.matched_context = value;
|
||||
}
|
||||
"ms" => config.matched_selected = value,
|
||||
"mc" => config.matched_context = value,
|
||||
"fn" => config.filename = value,
|
||||
|
||||
+279
-76
@@ -5,10 +5,14 @@
|
||||
|
||||
use crate::{Config, RegexMode};
|
||||
use onig::{
|
||||
EncodedBytes, Regex, RegexOptions, Region, SearchOptions, Syntax, SyntaxBehavior,
|
||||
SyntaxOperator,
|
||||
EncodedBytes, Error, MatchParam, Regex, RegexOptions, Region, SearchOptions, Syntax,
|
||||
SyntaxBehavior, SyntaxOperator,
|
||||
};
|
||||
use onig_sys::{OnigEncCtype_ONIGENC_CTYPE_WORD, OnigEncodingUTF8};
|
||||
use onig_sys::{
|
||||
ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS, ONIGERR_INVALID_BACKREF, ONIGERR_RETRY_LIMIT_IN_MATCH_OVER,
|
||||
ONIGERR_RETRY_LIMIT_IN_SEARCH_OVER, OnigEncCtype_ONIGENC_CTYPE_WORD, OnigEncodingUTF8,
|
||||
};
|
||||
use std::io;
|
||||
use uucore::error::{UResult, USimpleError};
|
||||
|
||||
pub struct Matcher<'a> {
|
||||
@@ -26,26 +30,30 @@ impl<'a> Matcher<'a> {
|
||||
}
|
||||
|
||||
/// Decide whether `line` matches and return the positions to highlight.
|
||||
pub fn match_line(&self, line: &[u8]) -> Option<Vec<(usize, usize)>> {
|
||||
///
|
||||
/// Returns an error if the regex engine bails out (e.g. it exceeds its
|
||||
/// backtracking retry limit on a pathological pattern); the caller turns
|
||||
/// that into a GNU-style diagnostic and exit code 2 rather than aborting.
|
||||
pub fn match_line(&self, line: &[u8]) -> io::Result<Option<Vec<(usize, usize)>>> {
|
||||
let mut any_seen = false;
|
||||
let positions: Vec<_> = MatchIter::new(&self.patterns, line)
|
||||
.filter(|&(start, end)| {
|
||||
any_seen = true;
|
||||
// Drop zero-length matches from the output.
|
||||
if start == end {
|
||||
return false;
|
||||
}
|
||||
// Drop matches that don't span the whole line if `-x` was requested.
|
||||
if self.config.line_regexp && !(start == 0 && end == line.len()) {
|
||||
return false;
|
||||
}
|
||||
// Drop matches that aren't word matches if `-w` was requested.
|
||||
if self.config.word_regexp && !Self::is_word_match(line, start, end) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
})
|
||||
.collect();
|
||||
let mut positions = Vec::new();
|
||||
let mut iter = MatchIter::new(&self.patterns, line).map_err(match_error)?;
|
||||
while let Some((start, end)) = iter.next_match().map_err(match_error)? {
|
||||
any_seen = true;
|
||||
// Drop zero-length matches from the output.
|
||||
if start == end {
|
||||
continue;
|
||||
}
|
||||
// Drop matches that don't span the whole line if `-x` was requested.
|
||||
if self.config.line_regexp && !(start == 0 && end == line.len()) {
|
||||
continue;
|
||||
}
|
||||
// Drop matches that aren't word matches if `-w` was requested.
|
||||
if self.config.word_regexp && !Self::is_word_match(line, start, end) {
|
||||
continue;
|
||||
}
|
||||
positions.push((start, end));
|
||||
}
|
||||
|
||||
let raw_matched = if self.config.line_regexp || self.config.word_regexp {
|
||||
// -w / -x are authoritative once positions are filtered.
|
||||
@@ -54,23 +62,25 @@ impl<'a> Matcher<'a> {
|
||||
any_seen
|
||||
};
|
||||
|
||||
if raw_matched != self.config.invert_match {
|
||||
Some(positions)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
Ok((raw_matched != self.config.invert_match).then_some(positions))
|
||||
}
|
||||
|
||||
/// Cheap match check that doesn't enumerate positions.
|
||||
pub fn is_match(&self, line: &[u8]) -> Option<Vec<(usize, usize)>> {
|
||||
pub fn is_match(&self, line: &[u8]) -> io::Result<Option<Vec<(usize, usize)>>> {
|
||||
// `-w` / `-x` need positions to filter, so we fall back to `match_line`.
|
||||
let matched = if self.config.line_regexp || self.config.word_regexp {
|
||||
self.match_line(line).is_some()
|
||||
self.match_line(line)?.is_some()
|
||||
} else {
|
||||
let raw_matched = self.patterns.iter().any(|p| p.is_match(line));
|
||||
let mut raw_matched = false;
|
||||
for p in &self.patterns {
|
||||
if p.is_match(line).map_err(match_error)? {
|
||||
raw_matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
raw_matched != self.config.invert_match
|
||||
};
|
||||
matched.then(Vec::new)
|
||||
Ok(matched.then(Vec::new))
|
||||
}
|
||||
|
||||
/// Word-boundary check `-w`.
|
||||
@@ -112,35 +122,31 @@ struct MatchIter<'a> {
|
||||
}
|
||||
|
||||
impl<'a> MatchIter<'a> {
|
||||
fn new(patterns: &'a [CompiledPattern], line: &'a [u8]) -> Self {
|
||||
Self {
|
||||
cursors: patterns
|
||||
.iter()
|
||||
.map(|pattern| {
|
||||
let mut c = Cursor {
|
||||
pattern,
|
||||
line,
|
||||
offset: 0,
|
||||
pending: None,
|
||||
};
|
||||
c.refill();
|
||||
c
|
||||
})
|
||||
.collect(),
|
||||
last_end: 0,
|
||||
fn new(patterns: &'a [CompiledPattern], line: &'a [u8]) -> Result<Self, Error> {
|
||||
let mut cursors = Vec::with_capacity(patterns.len());
|
||||
for pattern in patterns {
|
||||
let mut c = Cursor {
|
||||
pattern,
|
||||
line,
|
||||
offset: 0,
|
||||
pending: None,
|
||||
};
|
||||
c.refill()?;
|
||||
cursors.push(c);
|
||||
}
|
||||
Ok(Self {
|
||||
cursors,
|
||||
last_end: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Iterator for MatchIter<'a> {
|
||||
type Item = (usize, usize);
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
/// Yield the next match across all patterns, or `None` when exhausted.
|
||||
fn next_match(&mut self) -> Result<Option<(usize, usize)>, Error> {
|
||||
// Discard stale pendings that fall before the last emit.
|
||||
for cursor in &mut self.cursors {
|
||||
if matches!(cursor.pending, Some((s, _)) if s < self.last_end) {
|
||||
cursor.offset = self.last_end;
|
||||
cursor.refill();
|
||||
cursor.refill()?;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,12 +159,15 @@ impl<'a> Iterator for MatchIter<'a> {
|
||||
.enumerate()
|
||||
.filter_map(|(i, c)| c.pending.map(|p| (i, p)))
|
||||
.min_by_key(|&(_, (s, e))| (s, std::cmp::Reverse(e)))
|
||||
.map(|(i, _)| i)?;
|
||||
.map(|(i, _)| i);
|
||||
let Some(best_idx) = best_idx else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let (start, end) = self.cursors[best_idx].pending.unwrap();
|
||||
self.cursors[best_idx].refill();
|
||||
self.cursors[best_idx].refill()?;
|
||||
self.last_end = end;
|
||||
Some((start, end))
|
||||
Ok(Some((start, end)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -173,24 +182,25 @@ struct Cursor<'a> {
|
||||
}
|
||||
|
||||
impl Cursor<'_> {
|
||||
fn refill(&mut self) {
|
||||
fn refill(&mut self) -> Result<(), Error> {
|
||||
if self.offset >= self.line.len() {
|
||||
self.pending = None;
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
let Some((start, leftmost_end)) = self.pattern.search_leftmost(self.line, self.offset)
|
||||
let Some((start, leftmost_end)) = self.pattern.search_leftmost(self.line, self.offset)?
|
||||
else {
|
||||
self.pending = None;
|
||||
return;
|
||||
return Ok(());
|
||||
};
|
||||
let end = self
|
||||
.pattern
|
||||
.longest_end_at(self.line, start)
|
||||
.longest_end_at(self.line, start)?
|
||||
.unwrap_or(leftmost_end);
|
||||
// Advance the next search past the match we just found.
|
||||
// Zero-length matches need a +1 nudge to avoid spinning forever.
|
||||
self.offset = end.max(start + 1);
|
||||
self.pending = Some((start, end));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -205,6 +215,12 @@ struct CompiledPattern {
|
||||
|
||||
impl CompiledPattern {
|
||||
fn compile(pattern: &str, config: &Config) -> UResult<Self> {
|
||||
// GNU grep rejects the confusing `[:name:]` bracket form (a misspelled
|
||||
// `[[:name:]]`) in basic/extended modes; oniguruma accepts it silently.
|
||||
if matches!(config.regex_mode, RegexMode::Basic | RegexMode::Extended) {
|
||||
check_confusing_bracket(pattern)?;
|
||||
}
|
||||
|
||||
let mut syntax = *match config.regex_mode {
|
||||
RegexMode::Fixed => Syntax::asis(),
|
||||
RegexMode::Basic => Syntax::grep(),
|
||||
@@ -232,17 +248,29 @@ impl CompiledPattern {
|
||||
options |= RegexOptions::REGEX_OPTION_IGNORECASE;
|
||||
}
|
||||
|
||||
fn compile_with(pattern: &str, syntax: &Syntax, options: RegexOptions) -> UResult<Regex> {
|
||||
let mode = config.regex_mode;
|
||||
fn compile_with(
|
||||
pattern: &str,
|
||||
syntax: &Syntax,
|
||||
options: RegexOptions,
|
||||
mode: RegexMode,
|
||||
) -> UResult<Regex> {
|
||||
Regex::with_options_and_encoding(pattern, options, syntax).map_err(|err| {
|
||||
USimpleError::new(2, format!("invalid pattern \"{pattern}\": {err}"))
|
||||
// Prefer GNU grep's wording for the errors it has a dedicated
|
||||
// message for; fall back to oniguruma's text otherwise.
|
||||
match gnu_error_message(err.code(), mode) {
|
||||
Some(msg) => USimpleError::new(2, msg.to_string()),
|
||||
None => USimpleError::new(2, format!("invalid pattern \"{pattern}\": {err}")),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
let leftmost = compile_with(pattern, &syntax, options)?;
|
||||
let leftmost = compile_with(pattern, &syntax, options, mode)?;
|
||||
let longest_anchored = compile_with(
|
||||
pattern,
|
||||
&syntax,
|
||||
options | RegexOptions::REGEX_OPTION_FIND_LONGEST,
|
||||
mode,
|
||||
)?;
|
||||
Ok(Self {
|
||||
leftmost,
|
||||
@@ -251,41 +279,216 @@ impl CompiledPattern {
|
||||
}
|
||||
|
||||
/// Find the leftmost match starting at or after `offset`.
|
||||
fn search_leftmost(&self, line: &[u8], offset: usize) -> Option<(usize, usize)> {
|
||||
fn search_leftmost(&self, line: &[u8], offset: usize) -> Result<Option<(usize, usize)>, Error> {
|
||||
let mut region = Region::new();
|
||||
self.leftmost.search_with_encoding(
|
||||
let found = self.leftmost.search_with_param(
|
||||
EncodedBytes::from_parts(line, &raw mut OnigEncodingUTF8),
|
||||
offset,
|
||||
line.len(),
|
||||
SearchOptions::SEARCH_OPTION_NONE,
|
||||
Some(&mut region),
|
||||
MatchParam::default(),
|
||||
)?;
|
||||
region.pos(0)
|
||||
Ok(found.and_then(|_| region.pos(0)))
|
||||
}
|
||||
|
||||
/// Given a known leftmost start `start`, return the longest extent
|
||||
/// of a match anchored exactly there = POSIX leftmost-longest end.
|
||||
fn longest_end_at(&self, line: &[u8], start: usize) -> Option<usize> {
|
||||
fn longest_end_at(&self, line: &[u8], start: usize) -> Result<Option<usize>, Error> {
|
||||
let mut region = Region::new();
|
||||
self.longest_anchored.match_with_encoding(
|
||||
self.longest_anchored.match_with_param(
|
||||
EncodedBytes::from_parts(line, &raw mut OnigEncodingUTF8),
|
||||
start,
|
||||
SearchOptions::SEARCH_OPTION_NONE,
|
||||
Some(&mut region),
|
||||
);
|
||||
region.pos(0).map(|(_, end)| end)
|
||||
MatchParam::default(),
|
||||
)?;
|
||||
Ok(region.pos(0).map(|(_, end)| end))
|
||||
}
|
||||
|
||||
/// True if any match exists in `line` (including zero-length).
|
||||
fn is_match(&self, line: &[u8]) -> bool {
|
||||
self.leftmost
|
||||
.search_with_encoding(
|
||||
fn is_match(&self, line: &[u8]) -> Result<bool, Error> {
|
||||
Ok(self
|
||||
.leftmost
|
||||
.search_with_param(
|
||||
EncodedBytes::from_parts(line, &raw mut OnigEncodingUTF8),
|
||||
0,
|
||||
line.len(),
|
||||
SearchOptions::SEARCH_OPTION_NONE,
|
||||
None,
|
||||
)
|
||||
.is_some()
|
||||
MatchParam::default(),
|
||||
)?
|
||||
.is_some())
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a regex-engine match-time error into an I/O error carrying GNU
|
||||
/// grep's wording. The only error we expect in practice is the backtracking
|
||||
/// retry limit being exceeded on a pathological pattern; GNU reports this as
|
||||
/// `exceeded PCRE's backtracking limit` and exits 2 instead of aborting.
|
||||
fn match_error(err: Error) -> io::Error {
|
||||
let message = if matches!(
|
||||
err.code(),
|
||||
ONIGERR_RETRY_LIMIT_IN_MATCH_OVER | ONIGERR_RETRY_LIMIT_IN_SEARCH_OVER
|
||||
) {
|
||||
"exceeded PCRE's backtracking limit".to_string()
|
||||
} else {
|
||||
err.description().to_string()
|
||||
};
|
||||
io::Error::other(message)
|
||||
}
|
||||
|
||||
/// Map an oniguruma compile-error code to GNU grep's wording for the same
|
||||
/// condition, when one exists. GNU emits a bare POSIX-style diagnostic (e.g.
|
||||
/// `Invalid range end`) rather than oniguruma's phrasing, so translating keeps
|
||||
/// us byte-compatible. Returns `None` for errors with no GNU equivalent, where
|
||||
/// the caller falls back to oniguruma's own message.
|
||||
fn gnu_error_message(code: i32, mode: RegexMode) -> Option<&'static str> {
|
||||
match code {
|
||||
// e.g. `[b-a]`: a range whose end precedes its start.
|
||||
ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS => Some("Invalid range end"),
|
||||
// e.g. `(.)\2`: a back-reference to a group that does not exist. GNU
|
||||
// (via PCRE2) and gnulib's regex word this differently.
|
||||
ONIGERR_INVALID_BACKREF if mode == RegexMode::Perl => {
|
||||
Some("reference to non-existent subpattern")
|
||||
}
|
||||
ONIGERR_INVALID_BACKREF => Some("Invalid back reference"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Reject the confusing `[:name:]` bracket form the way GNU grep does.
|
||||
///
|
||||
/// A bracket expression like `[:space:]` is almost always a misspelled
|
||||
/// `[[:space:]]`; GNU grep flags it with a dedicated diagnostic and exits 2,
|
||||
/// whereas oniguruma silently treats it as the set `{':','s','p',…}`. This
|
||||
/// scans the pattern for that form and returns the same error.
|
||||
fn check_confusing_bracket(pattern: &str) -> UResult<()> {
|
||||
let bytes = pattern.as_bytes();
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
match bytes[i] {
|
||||
// Outside a bracket a backslash escapes the next character, so
|
||||
// `\[` does not open a bracket expression.
|
||||
b'\\' => i += 2,
|
||||
b'[' => {
|
||||
i += 1;
|
||||
if bracket_warns(bytes, &mut i) {
|
||||
return Err(USimpleError::new(
|
||||
2,
|
||||
"character class syntax is [[:space:]], not [:space:]".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
_ => i += 1,
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Consume a single bracket expression starting just past its opening `[` and
|
||||
/// report whether GNU grep's colon warning fires for it.
|
||||
///
|
||||
/// This is a faithful port of the `colon_warning_state` logic in GNU grep's
|
||||
/// `parse_bracket_exp` (gnulib `dfa.c`). The state is a bitmask:
|
||||
/// bit 0 — first character is a colon
|
||||
/// bit 1 — last character is a colon
|
||||
/// bit 2 — includes some other (non-colon) character
|
||||
/// bit 3 — includes a range, char/equivalence class, or collating element
|
||||
/// The warning fires exactly when the state ends equal to `7` (bits 0–2 set,
|
||||
/// bit 3 clear). On the way it advances `i` past the closing `]`.
|
||||
fn bracket_warns(bytes: &[u8], i: &mut usize) -> bool {
|
||||
fn fetch(bytes: &[u8], i: &mut usize) -> Option<u8> {
|
||||
let b = bytes.get(*i).copied();
|
||||
if b.is_some() {
|
||||
*i += 1;
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
let Some(first) = fetch(bytes, i) else {
|
||||
return false;
|
||||
};
|
||||
let mut c = first;
|
||||
if c == b'^' {
|
||||
match fetch(bytes, i) {
|
||||
Some(x) => c = x,
|
||||
None => return false,
|
||||
}
|
||||
}
|
||||
let mut state: u8 = u8::from(c == b':');
|
||||
|
||||
'scan: loop {
|
||||
state &= !2;
|
||||
let mut c1: Option<u8> = None;
|
||||
|
||||
if c == b'[' {
|
||||
let Some(nc1) = fetch(bytes, i) else {
|
||||
return false;
|
||||
};
|
||||
// `[:`, `[.` and `[=` introduce a class / collating / equivalence
|
||||
// element; consume it whole and mark bit 3.
|
||||
if nc1 == b':' || nc1 == b'.' || nc1 == b'=' {
|
||||
loop {
|
||||
match fetch(bytes, i) {
|
||||
None => break,
|
||||
Some(cc) if cc == nc1 && bytes.get(*i).copied() == Some(b']') => break,
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
if fetch(bytes, i).is_none() {
|
||||
return false; // consumes the `]`
|
||||
}
|
||||
state |= 8;
|
||||
match fetch(bytes, i) {
|
||||
Some(b']') => break 'scan,
|
||||
Some(x) => {
|
||||
c = x;
|
||||
continue 'scan;
|
||||
}
|
||||
None => return false,
|
||||
}
|
||||
}
|
||||
// Otherwise `[` is an ordinary character; `nc1` is the lookahead.
|
||||
c1 = Some(nc1);
|
||||
}
|
||||
|
||||
if c1.is_none() {
|
||||
c1 = fetch(bytes, i);
|
||||
}
|
||||
|
||||
if c1 == Some(b'-') {
|
||||
let Some(mut c2) = fetch(bytes, i) else {
|
||||
return false;
|
||||
};
|
||||
if c2 == b'[' && bytes.get(*i).copied() == Some(b'.') {
|
||||
c2 = b']';
|
||||
}
|
||||
if c2 == b']' {
|
||||
// `[x-]`: the hyphen is a literal; put the `]` back so the
|
||||
// loop terminator sees it next.
|
||||
*i -= 1;
|
||||
} else {
|
||||
state |= 8;
|
||||
match fetch(bytes, i) {
|
||||
Some(b']') => break 'scan,
|
||||
Some(x) => {
|
||||
c = x;
|
||||
continue 'scan;
|
||||
}
|
||||
None => return false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
state |= if c == b':' { 2 } else { 4 };
|
||||
|
||||
match c1 {
|
||||
Some(b']') => break 'scan,
|
||||
Some(x) => c = x,
|
||||
None => return false,
|
||||
}
|
||||
}
|
||||
|
||||
state == 7
|
||||
}
|
||||
|
||||
+3
-3
@@ -281,7 +281,7 @@ impl<'a> Searcher<'a> {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if let Some(positions) = self.session_match_line(line) {
|
||||
if let Some(positions) = self.session_match_line(line)? {
|
||||
// TODO: GNU grep respects LANG. Here, I'm always checking for valid UTF-8.
|
||||
if !self.session_mark_binary_if(|| std::str::from_utf8(line).is_err()) {
|
||||
return Ok(false);
|
||||
@@ -321,9 +321,9 @@ impl<'a> Searcher<'a> {
|
||||
self.config.binary_mode != BinaryMode::WithoutMatch
|
||||
}
|
||||
|
||||
fn session_match_line(&self, line: &[u8]) -> Option<Vec<(usize, usize)>> {
|
||||
fn session_match_line(&self, line: &[u8]) -> io::Result<Option<Vec<(usize, usize)>>> {
|
||||
if !self.session_can_match() {
|
||||
None
|
||||
Ok(None)
|
||||
} else if self.session_needs_match_positions() {
|
||||
self.matcher.match_line(line)
|
||||
} else {
|
||||
|
||||
@@ -126,6 +126,142 @@ fn ere_invalid_pattern_is_error() {
|
||||
.stderr_contains("invalid pattern");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confusing_bracket_class_is_error() {
|
||||
// GNU grep rejects the misspelled `[:name:]` form (meant to be
|
||||
// `[[:name:]]`) with a dedicated diagnostic and exit code 2.
|
||||
// No piped input: the pattern is rejected at compile time, before stdin is
|
||||
// read, so feeding stdin would race with the child exiting (broken pipe).
|
||||
for pattern in ["[:space:]", "[:digit:]", "[^:space:]", "x[:space:]y"] {
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&[pattern])
|
||||
.fails_with_code(2)
|
||||
.stderr_is("grep: character class syntax is [[:space:]], not [:space:]\n");
|
||||
}
|
||||
|
||||
// The same diagnostic applies in extended mode.
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["-E", "[:space:]"])
|
||||
.fails_with_code(2)
|
||||
.stderr_is("grep: character class syntax is [[:space:]], not [:space:]\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lookalike_brackets_are_not_confusing() {
|
||||
// Patterns that are NOT the confusing `[:name:]` form must compile
|
||||
// normally (no diagnostic). A proper class, a colon set, a range, a
|
||||
// trailing colon set, and `-F` literal text all stay valid.
|
||||
for pattern in [
|
||||
"[[:space:]]",
|
||||
"[::]",
|
||||
"[:space]",
|
||||
"[:spac-e:]",
|
||||
"[a:space:]",
|
||||
] {
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&[pattern])
|
||||
.pipe_in("z\n")
|
||||
.fails_with_code(1)
|
||||
.no_output();
|
||||
}
|
||||
|
||||
// `\[` does not open a bracket expression.
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["\\[:space:]"])
|
||||
.pipe_in("z\n")
|
||||
.fails_with_code(1)
|
||||
.no_output();
|
||||
|
||||
// `-F` treats the text literally, so no diagnostic.
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["-F", "[:space:]"])
|
||||
.pipe_in("x\n")
|
||||
.fails_with_code(1)
|
||||
.no_output();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reversed_range_uses_gnu_wording() {
|
||||
// A range like `[b-a]` is an error; GNU prints the bare POSIX diagnostic
|
||||
// "Invalid range end" (not oniguruma's phrasing) and exits 2.
|
||||
// No piped input: the pattern is rejected before stdin is read.
|
||||
for args in [&["[b-a]"][..], &["-E", "[b-a]"][..]] {
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(args)
|
||||
.fails_with_code(2)
|
||||
.stderr_is("grep: Invalid range end\n");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pcre_backtracking_limit_does_not_abort() {
|
||||
// A pathological PCRE pattern can exceed oniguruma's retry limit. GNU
|
||||
// grep reports this and exits 2 (it must not crash); stdout stays empty.
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["-P", "((a+)*)+$"])
|
||||
.pipe_in("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab\n")
|
||||
.fails_with_code(2)
|
||||
.stdout_is("")
|
||||
.stderr_contains("backtracking limit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_backreference_uses_gnu_wording() {
|
||||
// A back-reference to a non-existent group is worded differently by GNU
|
||||
// depending on the engine: PCRE (-P) vs gnulib regex (BRE/ERE).
|
||||
// No piped input: these patterns are rejected before stdin is read.
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["-P", r"(.)\2"])
|
||||
.fails_with_code(2)
|
||||
.stderr_is("grep: reference to non-existent subpattern\n");
|
||||
|
||||
for args in [&["-E", r"(.)\2"][..], &[r"\(.\)\2"][..]] {
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(args)
|
||||
.fails_with_code(2)
|
||||
.stderr_is("grep: Invalid back reference\n");
|
||||
}
|
||||
|
||||
// A valid back-reference with -Pw / -Px must still match.
|
||||
for flag in ["-Pw", "-Px"] {
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&[flag, r"(.)\1"])
|
||||
.pipe_in("aa\n")
|
||||
.succeeds()
|
||||
.stdout_is("aa\n");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grep_colors_mt_and_grep_color_deprecation() {
|
||||
// GREP_COLORS `mt` sets the match color (both selected and context).
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["--color=always", "."])
|
||||
.env("GREP_COLORS", "mt=36")
|
||||
.pipe_in("x\n")
|
||||
.succeeds()
|
||||
.stdout_is("\u{1b}[36m\u{1b}[Kx\u{1b}[m\u{1b}[K\n");
|
||||
|
||||
// GREP_COLOR is deprecated: it still sets the match color, but emits a
|
||||
// warning when color is actually produced.
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["--color=always", "."])
|
||||
.env("GREP_COLOR", "36")
|
||||
.pipe_in("x\n")
|
||||
.succeeds()
|
||||
.stdout_is("\u{1b}[36m\u{1b}[Kx\u{1b}[m\u{1b}[K\n")
|
||||
.stderr_is("grep: warning: GREP_COLOR='36' is deprecated; use GREP_COLORS='mt=36'\n");
|
||||
|
||||
// No warning when color output is disabled.
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["--color=never", "."])
|
||||
.env("GREP_COLOR", "36")
|
||||
.pipe_in("x\n")
|
||||
.succeeds()
|
||||
.stdout_is("x\n")
|
||||
.no_stderr();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixed_string_is_literal() {
|
||||
// Metacharacters are not interpreted.
|
||||
|
||||
Reference in New Issue
Block a user