5 Commits
10 changed files with 1081 additions and 497 deletions
+37
View File
@@ -0,0 +1,37 @@
name: CodSpeed
on:
push:
branches:
- "main"
pull_request:
# `workflow_dispatch` allows CodSpeed to trigger backtest
# performance analysis in order to generate initial data.
workflow_dispatch:
permissions:
contents: read
id-token: write
jobs:
codspeed:
name: Run benchmarks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup Rust toolchain, cache and cargo-codspeed binary
uses: moonrepo/setup-rust@v0
with:
channel: stable
cache-target: release
bins: cargo-codspeed
- name: Build the benchmark target(s)
run: cargo codspeed build
- name: Run the benchmarks
uses: CodSpeedHQ/action@v4
with:
mode: simulation
run: cargo codspeed run
+55
View File
@@ -0,0 +1,55 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
exclude: ^tests/fixtures/
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v6.0.0
hooks:
- id: check-added-large-files
- id: check-executables-have-shebangs
- id: check-json
exclude: '\.vscode/(cSpell|extensions)\.json' # cSpell.json and extensions.json use comments
- id: check-shebang-scripts-are-executable
exclude: '.+\.rs' # would be triggered by #![some_attribute]
- id: check-symlinks
- id: check-toml
- id: check-yaml
args: [ --allow-multiple-documents ]
- id: destroyed-symlinks
- id: end-of-file-fixer
- id: mixed-line-ending
args: [ --fix=lf ]
- id: trailing-whitespace
- repo: local
hooks:
- id: rust-linting
name: Rust linting
description: Run cargo fmt on files included in the commit.
entry: cargo +stable fmt --
pass_filenames: true
types: [file, rust]
language: system
- id: rust-clippy
name: Rust clippy
description: Run cargo clippy on files included in the commit.
entry: cargo +stable clippy --workspace --all-targets --all-features -- -D warnings
pass_filenames: false
types: [file, rust]
language: system
- id: cargo-lock-check
name: Cargo.lock sync check
description: Ensure Cargo.lock and fuzz/Cargo.lock are up-to-date.
entry: bash -c 'for dir in . fuzz; do if [ -d "$dir" ]; then ( cd "$dir" && cargo fetch --quiet ); fi; done'
pass_filenames: false
files: 'Cargo\.(toml|lock)$'
language: system
- id: cspell
name: Code spell checker (cspell)
description: Run cspell to check for spelling errors (if available).
entry: bash -c 'if command -v cspell >/dev/null 2>&1; then cspell --no-must-find-files -- "$@"; else echo "cspell not found, skipping spell check"; exit 0; fi' --
pass_filenames: true
language: system
ci:
skip: [rust-linting, rust-clippy, cargo-lock-check, cspell]
Generated
+532 -11
View File
File diff suppressed because it is too large Load Diff
+5
View File
@@ -27,5 +27,10 @@ onig_sys = { version = "*", default-features = false }
uucore = "0.8.0"
walkdir = "2.5"
[[bench]]
name = "grep_bench"
harness = false
[dev-dependencies]
criterion = { version = "4.7.0", package = "codspeed-criterion-compat" }
uutests = "0.8.0"
+6
View File
@@ -4,6 +4,7 @@
[![dependency status](https://deps.rs/repo/github/uutils/grep/status.svg)](https://deps.rs/repo/github/uutils/grep)
[![CodeCov](https://codecov.io/gh/uutils/grep/branch/main/graph/badge.svg)](https://codecov.io/gh/uutils/grep)
[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/uutils/grep?utm_source=badge)
# Grep, now in Rust
@@ -29,10 +30,15 @@ cargo build --release
cargo test
```
## Pre-commit hooks
This project uses [pre-commit](https://pre-commit.com); run `pre-commit install` to enable the git hooks.
## Known Issues
* Does not take `LANG`, etc., into account for handling file encodings (non-UTF8 matches are treated as binary)
* No localization support yet
* Performances need to be improved
## Contributing
+288
View File
@@ -0,0 +1,288 @@
// This file is part of the uutils grep package.
//
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.
use criterion::{Criterion, black_box, criterion_group, criterion_main};
use uu_grep::matcher::Matcher;
use uu_grep::{BinaryMode, ColorConfig, Config, DeviceMode, DirectoryMode, GlobSet, RegexMode};
fn make_config<'a>(
patterns: &'a [&'a str],
regex_mode: RegexMode,
ignore_case: bool,
invert_match: bool,
word_regexp: bool,
) -> Config<'a> {
Config {
directory_mode: DirectoryMode::Read,
device_mode: DeviceMode::Default,
follow_symlinks: false,
include_globs: GlobSet::new(),
exclude_globs: GlobSet::new(),
exclude_dir_globs: GlobSet::new(),
label: "(standard input)",
#[cfg(windows)]
strip_cr: false,
binary_mode: BinaryMode::Binary,
max_count: None,
before_context: 0,
after_context: 0,
has_context: false,
patterns,
regex_mode,
ignore_case,
invert_match,
word_regexp,
line_regexp: false,
quiet: true,
count: false,
show_filename: false,
files_with_matches: false,
files_without_match: false,
only_matching: false,
byte_offset: false,
line_number: false,
initial_tab: false,
null_separator: false,
null_data: false,
line_buffered: false,
no_messages: true,
group_separator: None,
use_color: false,
color_config: ColorConfig {
matched_selected: "",
matched_context: "",
filename: "",
line_number: "",
byte_offset: "",
separator: "",
selected_line: "",
context_line: "",
reverse_video: false,
no_erase: false,
},
}
}
fn bench_compile(c: &mut Criterion) {
let mut group = c.benchmark_group("compile");
group.bench_function("fixed_string", |b| {
b.iter(|| {
let patterns: &[&str] = &["hello world"];
let config = make_config(patterns, RegexMode::Fixed, false, false, false);
let matcher = Matcher::compile(black_box(&config)).unwrap();
let _ = black_box(&matcher);
})
});
group.bench_function("basic_regex", |b| {
b.iter(|| {
let patterns: &[&str] = &[r"[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}"];
let config = make_config(patterns, RegexMode::Basic, false, false, false);
let matcher = Matcher::compile(black_box(&config)).unwrap();
let _ = black_box(&matcher);
})
});
group.bench_function("extended_regex", |b| {
b.iter(|| {
let patterns: &[&str] = &[r"[0-9]{4}-[0-9]{2}-[0-9]{2}"];
let config = make_config(patterns, RegexMode::Extended, false, false, false);
let matcher = Matcher::compile(black_box(&config)).unwrap();
let _ = black_box(&matcher);
})
});
group.bench_function("perl_regex", |b| {
b.iter(|| {
let patterns: &[&str] = &[r"\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}"];
let config = make_config(patterns, RegexMode::Perl, false, false, false);
let matcher = Matcher::compile(black_box(&config)).unwrap();
let _ = black_box(&matcher);
})
});
group.bench_function("multiple_patterns", |b| {
b.iter(|| {
let patterns: &[&str] = &["error", "warning", "critical", "fatal", "panic"];
let config = make_config(patterns, RegexMode::Fixed, false, false, false);
let matcher = Matcher::compile(black_box(&config)).unwrap();
let _ = black_box(&matcher);
})
});
group.finish();
}
fn bench_match(c: &mut Criterion) {
let mut group = c.benchmark_group("match");
// Fixed string match - hit
{
let patterns: &[&str] = &["ERROR"];
let config = make_config(patterns, RegexMode::Fixed, false, false, false);
let matcher = Matcher::compile(&config).unwrap();
let line = b"2024-01-15 10:30:45 ERROR: Connection timeout on server-42";
group.bench_function("fixed_string_hit", |b| {
b.iter(|| black_box(matcher.match_line(black_box(line))))
});
}
// Fixed string match - miss
{
let patterns: &[&str] = &["CRITICAL"];
let config = make_config(patterns, RegexMode::Fixed, false, false, false);
let matcher = Matcher::compile(&config).unwrap();
let line = b"2024-01-15 10:30:45 INFO: Server started successfully";
group.bench_function("fixed_string_miss", |b| {
b.iter(|| black_box(matcher.match_line(black_box(line))))
});
}
// Extended regex match
{
let patterns: &[&str] = &[r"[0-9]{4}-[0-9]{2}-[0-9]{2}"];
let config = make_config(patterns, RegexMode::Extended, false, false, false);
let matcher = Matcher::compile(&config).unwrap();
let line = b"2024-01-15 10:30:45 ERROR: Connection timeout";
group.bench_function("extended_regex_hit", |b| {
b.iter(|| black_box(matcher.match_line(black_box(line))))
});
}
// Case-insensitive match
{
let patterns: &[&str] = &["error"];
let config = make_config(patterns, RegexMode::Fixed, true, false, false);
let matcher = Matcher::compile(&config).unwrap();
let line = b"2024-01-15 10:30:45 ERROR: Connection timeout";
group.bench_function("case_insensitive_hit", |b| {
b.iter(|| black_box(matcher.match_line(black_box(line))))
});
}
// Inverted match
{
let patterns: &[&str] = &["ERROR"];
let config = make_config(patterns, RegexMode::Fixed, false, true, false);
let matcher = Matcher::compile(&config).unwrap();
let line = b"2024-01-15 10:30:45 INFO: Server started successfully";
group.bench_function("inverted_match", |b| {
b.iter(|| black_box(matcher.match_line(black_box(line))))
});
}
// Word boundary match
{
let patterns: &[&str] = &["error"];
let config = make_config(patterns, RegexMode::Fixed, true, false, true);
let matcher = Matcher::compile(&config).unwrap();
let line = b"2024-01-15 10:30:45 error: Connection timeout";
group.bench_function("word_boundary_hit", |b| {
b.iter(|| black_box(matcher.match_line(black_box(line))))
});
}
// Multiple patterns
{
let patterns: &[&str] = &["error", "warning", "critical", "fatal", "panic"];
let config = make_config(patterns, RegexMode::Fixed, true, false, false);
let matcher = Matcher::compile(&config).unwrap();
let line = b"2024-01-15 10:30:45 WARNING: High memory usage detected on node-7";
group.bench_function("multi_pattern_hit", |b| {
b.iter(|| black_box(matcher.match_line(black_box(line))))
});
}
// Long line
{
let patterns: &[&str] = &["needle"];
let config = make_config(patterns, RegexMode::Fixed, false, false, false);
let matcher = Matcher::compile(&config).unwrap();
let mut long_line = "a".repeat(5000);
long_line.push_str("needle");
long_line.push_str(&"b".repeat(5000));
let long_line_bytes = long_line.into_bytes();
group.bench_function("long_line_hit", |b| {
b.iter(|| black_box(matcher.match_line(black_box(&long_line_bytes))))
});
}
group.finish();
}
fn bench_throughput(c: &mut Criterion) {
let mut group = c.benchmark_group("throughput");
// Simulate processing many lines (like searching a log file)
let lines: Vec<Vec<u8>> = (0..1000)
.map(|i| {
if i % 50 == 0 {
format!(
"2024-01-15 10:30:{:02} ERROR: Connection timeout on server-{}",
i % 60,
i
)
.into_bytes()
} else {
format!(
"2024-01-15 10:30:{:02} INFO: Request processed in {}ms",
i % 60,
i * 3
)
.into_bytes()
}
})
.collect();
{
let patterns: &[&str] = &["ERROR"];
let config = make_config(patterns, RegexMode::Fixed, false, false, false);
let matcher = Matcher::compile(&config).unwrap();
group.bench_function("scan_1000_lines_fixed", |b| {
b.iter(|| {
let mut matches = 0u64;
for line in &lines {
if matcher.match_line(black_box(line)).is_some() {
matches += 1;
}
}
black_box(matches)
})
});
}
{
let patterns: &[&str] = &[r"[0-9]+ *ms"];
let config = make_config(patterns, RegexMode::Extended, false, false, false);
let matcher = Matcher::compile(&config).unwrap();
group.bench_function("scan_1000_lines_regex", |b| {
b.iter(|| {
let mut matches = 0u64;
for line in &lines {
if matcher.match_line(black_box(line)).is_some() {
matches += 1;
}
}
black_box(matches)
})
});
}
group.finish();
}
criterion_group!(benches, bench_compile, bench_match, bench_throughput);
criterion_main!(benches);
+79 -68
View File
@@ -3,9 +3,12 @@
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.
mod context_buffer;
mod line_buffer;
mod matcher;
#[doc(hidden)]
pub mod context_buffer;
#[doc(hidden)]
pub mod line_buffer;
#[doc(hidden)]
pub mod matcher;
mod output;
mod searcher;
@@ -20,7 +23,8 @@ use std::path::Path;
use uucore::error::{FromIo, UResult, USimpleError};
#[derive(Clone, Copy, PartialEq, Eq)]
enum RegexMode {
#[doc(hidden)]
pub enum RegexMode {
Fixed,
Basic,
Extended,
@@ -28,7 +32,8 @@ enum RegexMode {
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum BinaryMode {
#[doc(hidden)]
pub enum BinaryMode {
Binary,
Text,
WithoutMatch,
@@ -42,79 +47,84 @@ enum ColorMode {
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum DirectoryMode {
#[doc(hidden)]
pub enum DirectoryMode {
Read,
Skip,
Recurse,
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum DeviceMode {
#[doc(hidden)]
pub enum DeviceMode {
Default,
Read,
Skip,
}
struct ColorConfig<'a> {
matched_selected: &'a str,
matched_context: &'a str,
filename: &'a str,
line_number: &'a str,
byte_offset: &'a str,
separator: &'a str,
selected_line: &'a str,
context_line: &'a str,
#[doc(hidden)]
pub struct ColorConfig<'a> {
pub matched_selected: &'a str,
pub matched_context: &'a str,
pub filename: &'a str,
pub line_number: &'a str,
pub byte_offset: &'a str,
pub separator: &'a str,
pub selected_line: &'a str,
pub context_line: &'a str,
reverse_video: bool,
no_erase: bool,
pub reverse_video: bool,
pub no_erase: bool,
}
struct GlobSet {
#[doc(hidden)]
pub struct GlobSet {
patterns: Vec<glob::Pattern>,
}
struct Config<'a> {
#[doc(hidden)]
pub struct Config<'a> {
// Searcher
directory_mode: DirectoryMode,
device_mode: DeviceMode,
follow_symlinks: bool,
include_globs: GlobSet,
exclude_globs: GlobSet,
exclude_dir_globs: GlobSet,
label: &'a str,
pub directory_mode: DirectoryMode,
pub device_mode: DeviceMode,
pub follow_symlinks: bool,
pub include_globs: GlobSet,
pub exclude_globs: GlobSet,
pub exclude_dir_globs: GlobSet,
pub label: &'a str,
#[cfg(windows)]
strip_cr: bool,
binary_mode: BinaryMode,
max_count: Option<u64>,
before_context: usize,
after_context: usize,
has_context: bool,
pub strip_cr: bool,
pub binary_mode: BinaryMode,
pub max_count: Option<u64>,
pub before_context: usize,
pub after_context: usize,
pub has_context: bool,
// Matcher
patterns: &'a [&'a str],
regex_mode: RegexMode,
ignore_case: bool,
invert_match: bool,
word_regexp: bool,
line_regexp: bool,
pub patterns: &'a [&'a str],
pub regex_mode: RegexMode,
pub ignore_case: bool,
pub invert_match: bool,
pub word_regexp: bool,
pub line_regexp: bool,
// Output
quiet: bool,
count: bool,
show_filename: bool,
files_with_matches: bool,
files_without_match: bool,
only_matching: bool,
byte_offset: bool,
line_number: bool,
initial_tab: bool,
null_separator: bool,
null_data: bool,
line_buffered: bool,
no_messages: bool,
group_separator: Option<&'a str>,
use_color: bool,
color_config: ColorConfig<'a>,
pub quiet: bool,
pub count: bool,
pub show_filename: bool,
pub files_with_matches: bool,
pub files_without_match: bool,
pub only_matching: bool,
pub byte_offset: bool,
pub line_number: bool,
pub initial_tab: bool,
pub null_separator: bool,
pub null_data: bool,
pub line_buffered: bool,
pub no_messages: bool,
pub group_separator: Option<&'a str>,
pub use_color: bool,
pub color_config: ColorConfig<'a>,
}
#[uucore::main(no_signals)]
@@ -342,13 +352,6 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
ColorMode::Never => false,
ColorMode::Auto => std::io::stdout().is_terminal(),
};
// GREP_COLOR is deprecated in favour of GREP_COLORS' `mt` capability;
// GNU warns about it, but only when color output is actually produced.
if use_color && !grep_color.is_empty() {
eprintln!(
"grep: warning: GREP_COLOR='{grep_color}' is deprecated; use GREP_COLORS='mt={grep_color}'"
);
}
let color_config = ColorConfig::from_env(&grep_color, &grep_colors);
let config = Config {
@@ -861,8 +864,21 @@ fn expand_num_shorthand(args: impl Iterator<Item = OsString>) -> Vec<OsString> {
out
}
impl Default for GlobSet {
fn default() -> Self {
Self::new()
}
}
impl GlobSet {
fn with_capacity(capacity: usize) -> Self {
/// Create an empty GlobSet.
pub fn new() -> Self {
Self {
patterns: Vec::new(),
}
}
pub fn with_capacity(capacity: usize) -> Self {
Self {
patterns: Vec::with_capacity(capacity),
}
@@ -910,11 +926,6 @@ impl<'a> ColorConfig<'a> {
for item in grep_colors.split(':') {
if let Some((key, value)) = item.split_once('=') {
match key {
// `mt` sets both the selected- and context-match colors.
"mt" => {
config.matched_selected = value;
config.matched_context = value;
}
"ms" => config.matched_selected = value,
"mc" => config.matched_context = value,
"fn" => config.filename = value,
+76 -279
View File
@@ -5,14 +5,10 @@
use crate::{Config, RegexMode};
use onig::{
EncodedBytes, Error, MatchParam, Regex, RegexOptions, Region, SearchOptions, Syntax,
SyntaxBehavior, SyntaxOperator,
EncodedBytes, Regex, RegexOptions, Region, SearchOptions, Syntax, SyntaxBehavior,
SyntaxOperator,
};
use onig_sys::{
ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS, ONIGERR_INVALID_BACKREF, ONIGERR_RETRY_LIMIT_IN_MATCH_OVER,
ONIGERR_RETRY_LIMIT_IN_SEARCH_OVER, OnigEncCtype_ONIGENC_CTYPE_WORD, OnigEncodingUTF8,
};
use std::io;
use onig_sys::{OnigEncCtype_ONIGENC_CTYPE_WORD, OnigEncodingUTF8};
use uucore::error::{UResult, USimpleError};
pub struct Matcher<'a> {
@@ -30,30 +26,26 @@ impl<'a> Matcher<'a> {
}
/// Decide whether `line` matches and return the positions to highlight.
///
/// Returns an error if the regex engine bails out (e.g. it exceeds its
/// backtracking retry limit on a pathological pattern); the caller turns
/// that into a GNU-style diagnostic and exit code 2 rather than aborting.
pub fn match_line(&self, line: &[u8]) -> io::Result<Option<Vec<(usize, usize)>>> {
pub fn match_line(&self, line: &[u8]) -> Option<Vec<(usize, usize)>> {
let mut any_seen = false;
let mut positions = Vec::new();
let mut iter = MatchIter::new(&self.patterns, line).map_err(match_error)?;
while let Some((start, end)) = iter.next_match().map_err(match_error)? {
any_seen = true;
// Drop zero-length matches from the output.
if start == end {
continue;
}
// Drop matches that don't span the whole line if `-x` was requested.
if self.config.line_regexp && !(start == 0 && end == line.len()) {
continue;
}
// Drop matches that aren't word matches if `-w` was requested.
if self.config.word_regexp && !Self::is_word_match(line, start, end) {
continue;
}
positions.push((start, end));
}
let positions: Vec<_> = MatchIter::new(&self.patterns, line)
.filter(|&(start, end)| {
any_seen = true;
// Drop zero-length matches from the output.
if start == end {
return false;
}
// Drop matches that don't span the whole line if `-x` was requested.
if self.config.line_regexp && !(start == 0 && end == line.len()) {
return false;
}
// Drop matches that aren't word matches if `-w` was requested.
if self.config.word_regexp && !Self::is_word_match(line, start, end) {
return false;
}
true
})
.collect();
let raw_matched = if self.config.line_regexp || self.config.word_regexp {
// -w / -x are authoritative once positions are filtered.
@@ -62,25 +54,23 @@ impl<'a> Matcher<'a> {
any_seen
};
Ok((raw_matched != self.config.invert_match).then_some(positions))
if raw_matched != self.config.invert_match {
Some(positions)
} else {
None
}
}
/// Cheap match check that doesn't enumerate positions.
pub fn is_match(&self, line: &[u8]) -> io::Result<Option<Vec<(usize, usize)>>> {
pub fn is_match(&self, line: &[u8]) -> Option<Vec<(usize, usize)>> {
// `-w` / `-x` need positions to filter, so we fall back to `match_line`.
let matched = if self.config.line_regexp || self.config.word_regexp {
self.match_line(line)?.is_some()
self.match_line(line).is_some()
} else {
let mut raw_matched = false;
for p in &self.patterns {
if p.is_match(line).map_err(match_error)? {
raw_matched = true;
break;
}
}
let raw_matched = self.patterns.iter().any(|p| p.is_match(line));
raw_matched != self.config.invert_match
};
Ok(matched.then(Vec::new))
matched.then(Vec::new)
}
/// Word-boundary check `-w`.
@@ -122,31 +112,35 @@ struct MatchIter<'a> {
}
impl<'a> MatchIter<'a> {
fn new(patterns: &'a [CompiledPattern], line: &'a [u8]) -> Result<Self, Error> {
let mut cursors = Vec::with_capacity(patterns.len());
for pattern in patterns {
let mut c = Cursor {
pattern,
line,
offset: 0,
pending: None,
};
c.refill()?;
cursors.push(c);
}
Ok(Self {
cursors,
fn new(patterns: &'a [CompiledPattern], line: &'a [u8]) -> Self {
Self {
cursors: patterns
.iter()
.map(|pattern| {
let mut c = Cursor {
pattern,
line,
offset: 0,
pending: None,
};
c.refill();
c
})
.collect(),
last_end: 0,
})
}
}
}
/// Yield the next match across all patterns, or `None` when exhausted.
fn next_match(&mut self) -> Result<Option<(usize, usize)>, Error> {
impl<'a> Iterator for MatchIter<'a> {
type Item = (usize, usize);
fn next(&mut self) -> Option<Self::Item> {
// Discard stale pendings that fall before the last emit.
for cursor in &mut self.cursors {
if matches!(cursor.pending, Some((s, _)) if s < self.last_end) {
cursor.offset = self.last_end;
cursor.refill()?;
cursor.refill();
}
}
@@ -159,15 +153,12 @@ impl<'a> MatchIter<'a> {
.enumerate()
.filter_map(|(i, c)| c.pending.map(|p| (i, p)))
.min_by_key(|&(_, (s, e))| (s, std::cmp::Reverse(e)))
.map(|(i, _)| i);
let Some(best_idx) = best_idx else {
return Ok(None);
};
.map(|(i, _)| i)?;
let (start, end) = self.cursors[best_idx].pending.unwrap();
self.cursors[best_idx].refill()?;
self.cursors[best_idx].refill();
self.last_end = end;
Ok(Some((start, end)))
Some((start, end))
}
}
@@ -182,25 +173,24 @@ struct Cursor<'a> {
}
impl Cursor<'_> {
fn refill(&mut self) -> Result<(), Error> {
fn refill(&mut self) {
if self.offset >= self.line.len() {
self.pending = None;
return Ok(());
return;
}
let Some((start, leftmost_end)) = self.pattern.search_leftmost(self.line, self.offset)?
let Some((start, leftmost_end)) = self.pattern.search_leftmost(self.line, self.offset)
else {
self.pending = None;
return Ok(());
return;
};
let end = self
.pattern
.longest_end_at(self.line, start)?
.longest_end_at(self.line, start)
.unwrap_or(leftmost_end);
// Advance the next search past the match we just found.
// Zero-length matches need a +1 nudge to avoid spinning forever.
self.offset = end.max(start + 1);
self.pending = Some((start, end));
Ok(())
}
}
@@ -215,12 +205,6 @@ struct CompiledPattern {
impl CompiledPattern {
fn compile(pattern: &str, config: &Config) -> UResult<Self> {
// GNU grep rejects the confusing `[:name:]` bracket form (a misspelled
// `[[:name:]]`) in basic/extended modes; oniguruma accepts it silently.
if matches!(config.regex_mode, RegexMode::Basic | RegexMode::Extended) {
check_confusing_bracket(pattern)?;
}
let mut syntax = *match config.regex_mode {
RegexMode::Fixed => Syntax::asis(),
RegexMode::Basic => Syntax::grep(),
@@ -248,29 +232,17 @@ impl CompiledPattern {
options |= RegexOptions::REGEX_OPTION_IGNORECASE;
}
let mode = config.regex_mode;
fn compile_with(
pattern: &str,
syntax: &Syntax,
options: RegexOptions,
mode: RegexMode,
) -> UResult<Regex> {
fn compile_with(pattern: &str, syntax: &Syntax, options: RegexOptions) -> UResult<Regex> {
Regex::with_options_and_encoding(pattern, options, syntax).map_err(|err| {
// Prefer GNU grep's wording for the errors it has a dedicated
// message for; fall back to oniguruma's text otherwise.
match gnu_error_message(err.code(), mode) {
Some(msg) => USimpleError::new(2, msg.to_string()),
None => USimpleError::new(2, format!("invalid pattern \"{pattern}\": {err}")),
}
USimpleError::new(2, format!("invalid pattern \"{pattern}\": {err}"))
})
}
let leftmost = compile_with(pattern, &syntax, options, mode)?;
let leftmost = compile_with(pattern, &syntax, options)?;
let longest_anchored = compile_with(
pattern,
&syntax,
options | RegexOptions::REGEX_OPTION_FIND_LONGEST,
mode,
)?;
Ok(Self {
leftmost,
@@ -279,216 +251,41 @@ impl CompiledPattern {
}
/// Find the leftmost match starting at or after `offset`.
fn search_leftmost(&self, line: &[u8], offset: usize) -> Result<Option<(usize, usize)>, Error> {
fn search_leftmost(&self, line: &[u8], offset: usize) -> Option<(usize, usize)> {
let mut region = Region::new();
let found = self.leftmost.search_with_param(
self.leftmost.search_with_encoding(
EncodedBytes::from_parts(line, &raw mut OnigEncodingUTF8),
offset,
line.len(),
SearchOptions::SEARCH_OPTION_NONE,
Some(&mut region),
MatchParam::default(),
)?;
Ok(found.and_then(|_| region.pos(0)))
region.pos(0)
}
/// Given a known leftmost start `start`, return the longest extent
/// of a match anchored exactly there = POSIX leftmost-longest end.
fn longest_end_at(&self, line: &[u8], start: usize) -> Result<Option<usize>, Error> {
fn longest_end_at(&self, line: &[u8], start: usize) -> Option<usize> {
let mut region = Region::new();
self.longest_anchored.match_with_param(
self.longest_anchored.match_with_encoding(
EncodedBytes::from_parts(line, &raw mut OnigEncodingUTF8),
start,
SearchOptions::SEARCH_OPTION_NONE,
Some(&mut region),
MatchParam::default(),
)?;
Ok(region.pos(0).map(|(_, end)| end))
);
region.pos(0).map(|(_, end)| end)
}
/// True if any match exists in `line` (including zero-length).
fn is_match(&self, line: &[u8]) -> Result<bool, Error> {
Ok(self
.leftmost
.search_with_param(
fn is_match(&self, line: &[u8]) -> bool {
self.leftmost
.search_with_encoding(
EncodedBytes::from_parts(line, &raw mut OnigEncodingUTF8),
0,
line.len(),
SearchOptions::SEARCH_OPTION_NONE,
None,
MatchParam::default(),
)?
.is_some())
)
.is_some()
}
}
/// Convert a regex-engine match-time error into an I/O error carrying GNU
/// grep's wording. The only error we expect in practice is the backtracking
/// retry limit being exceeded on a pathological pattern; GNU reports this as
/// `exceeded PCRE's backtracking limit` and exits 2 instead of aborting.
fn match_error(err: Error) -> io::Error {
let message = if matches!(
err.code(),
ONIGERR_RETRY_LIMIT_IN_MATCH_OVER | ONIGERR_RETRY_LIMIT_IN_SEARCH_OVER
) {
"exceeded PCRE's backtracking limit".to_string()
} else {
err.description().to_string()
};
io::Error::other(message)
}
/// Map an oniguruma compile-error code to GNU grep's wording for the same
/// condition, when one exists. GNU emits a bare POSIX-style diagnostic (e.g.
/// `Invalid range end`) rather than oniguruma's phrasing, so translating keeps
/// us byte-compatible. Returns `None` for errors with no GNU equivalent, where
/// the caller falls back to oniguruma's own message.
fn gnu_error_message(code: i32, mode: RegexMode) -> Option<&'static str> {
match code {
// e.g. `[b-a]`: a range whose end precedes its start.
ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS => Some("Invalid range end"),
// e.g. `(.)\2`: a back-reference to a group that does not exist. GNU
// (via PCRE2) and gnulib's regex word this differently.
ONIGERR_INVALID_BACKREF if mode == RegexMode::Perl => {
Some("reference to non-existent subpattern")
}
ONIGERR_INVALID_BACKREF => Some("Invalid back reference"),
_ => None,
}
}
/// Reject the confusing `[:name:]` bracket form the way GNU grep does.
///
/// A bracket expression like `[:space:]` is almost always a misspelled
/// `[[:space:]]`; GNU grep flags it with a dedicated diagnostic and exits 2,
/// whereas oniguruma silently treats it as the set `{':','s','p',…}`. This
/// scans the pattern for that form and returns the same error.
fn check_confusing_bracket(pattern: &str) -> UResult<()> {
let bytes = pattern.as_bytes();
let mut i = 0;
while i < bytes.len() {
match bytes[i] {
// Outside a bracket a backslash escapes the next character, so
// `\[` does not open a bracket expression.
b'\\' => i += 2,
b'[' => {
i += 1;
if bracket_warns(bytes, &mut i) {
return Err(USimpleError::new(
2,
"character class syntax is [[:space:]], not [:space:]".to_string(),
));
}
}
_ => i += 1,
}
}
Ok(())
}
/// Consume a single bracket expression starting just past its opening `[` and
/// report whether GNU grep's colon warning fires for it.
///
/// This is a faithful port of the `colon_warning_state` logic in GNU grep's
/// `parse_bracket_exp` (gnulib `dfa.c`). The state is a bitmask:
/// bit 0 — first character is a colon
/// bit 1 — last character is a colon
/// bit 2 — includes some other (non-colon) character
/// bit 3 — includes a range, char/equivalence class, or collating element
/// The warning fires exactly when the state ends equal to `7` (bits 0–2 set,
/// bit 3 clear). On the way it advances `i` past the closing `]`.
fn bracket_warns(bytes: &[u8], i: &mut usize) -> bool {
fn fetch(bytes: &[u8], i: &mut usize) -> Option<u8> {
let b = bytes.get(*i).copied();
if b.is_some() {
*i += 1;
}
b
}
let Some(first) = fetch(bytes, i) else {
return false;
};
let mut c = first;
if c == b'^' {
match fetch(bytes, i) {
Some(x) => c = x,
None => return false,
}
}
let mut state: u8 = u8::from(c == b':');
'scan: loop {
state &= !2;
let mut c1: Option<u8> = None;
if c == b'[' {
let Some(nc1) = fetch(bytes, i) else {
return false;
};
// `[:`, `[.` and `[=` introduce a class / collating / equivalence
// element; consume it whole and mark bit 3.
if nc1 == b':' || nc1 == b'.' || nc1 == b'=' {
loop {
match fetch(bytes, i) {
None => break,
Some(cc) if cc == nc1 && bytes.get(*i).copied() == Some(b']') => break,
Some(_) => {}
}
}
if fetch(bytes, i).is_none() {
return false; // consumes the `]`
}
state |= 8;
match fetch(bytes, i) {
Some(b']') => break 'scan,
Some(x) => {
c = x;
continue 'scan;
}
None => return false,
}
}
// Otherwise `[` is an ordinary character; `nc1` is the lookahead.
c1 = Some(nc1);
}
if c1.is_none() {
c1 = fetch(bytes, i);
}
if c1 == Some(b'-') {
let Some(mut c2) = fetch(bytes, i) else {
return false;
};
if c2 == b'[' && bytes.get(*i).copied() == Some(b'.') {
c2 = b']';
}
if c2 == b']' {
// `[x-]`: the hyphen is a literal; put the `]` back so the
// loop terminator sees it next.
*i -= 1;
} else {
state |= 8;
match fetch(bytes, i) {
Some(b']') => break 'scan,
Some(x) => {
c = x;
continue 'scan;
}
None => return false,
}
}
}
state |= if c == b':' { 2 } else { 4 };
match c1 {
Some(b']') => break 'scan,
Some(x) => c = x,
None => return false,
}
}
state == 7
}
+3 -3
View File
@@ -281,7 +281,7 @@ impl<'a> Searcher<'a> {
return Ok(false);
}
if let Some(positions) = self.session_match_line(line)? {
if let Some(positions) = self.session_match_line(line) {
// TODO: GNU grep respects LANG. Here, I'm always checking for valid UTF-8.
if !self.session_mark_binary_if(|| std::str::from_utf8(line).is_err()) {
return Ok(false);
@@ -321,9 +321,9 @@ impl<'a> Searcher<'a> {
self.config.binary_mode != BinaryMode::WithoutMatch
}
fn session_match_line(&self, line: &[u8]) -> io::Result<Option<Vec<(usize, usize)>>> {
fn session_match_line(&self, line: &[u8]) -> Option<Vec<(usize, usize)>> {
if !self.session_can_match() {
Ok(None)
None
} else if self.session_needs_match_positions() {
self.matcher.match_line(line)
} else {
-136
View File
@@ -126,142 +126,6 @@ fn ere_invalid_pattern_is_error() {
.stderr_contains("invalid pattern");
}
#[test]
fn confusing_bracket_class_is_error() {
// GNU grep rejects the misspelled `[:name:]` form (meant to be
// `[[:name:]]`) with a dedicated diagnostic and exit code 2.
// No piped input: the pattern is rejected at compile time, before stdin is
// read, so feeding stdin would race with the child exiting (broken pipe).
for pattern in ["[:space:]", "[:digit:]", "[^:space:]", "x[:space:]y"] {
let (_s, mut c) = ucmd();
c.args(&[pattern])
.fails_with_code(2)
.stderr_is("grep: character class syntax is [[:space:]], not [:space:]\n");
}
// The same diagnostic applies in extended mode.
let (_s, mut c) = ucmd();
c.args(&["-E", "[:space:]"])
.fails_with_code(2)
.stderr_is("grep: character class syntax is [[:space:]], not [:space:]\n");
}
#[test]
fn lookalike_brackets_are_not_confusing() {
// Patterns that are NOT the confusing `[:name:]` form must compile
// normally (no diagnostic). A proper class, a colon set, a range, a
// trailing colon set, and `-F` literal text all stay valid.
for pattern in [
"[[:space:]]",
"[::]",
"[:space]",
"[:spac-e:]",
"[a:space:]",
] {
let (_s, mut c) = ucmd();
c.args(&[pattern])
.pipe_in("z\n")
.fails_with_code(1)
.no_output();
}
// `\[` does not open a bracket expression.
let (_s, mut c) = ucmd();
c.args(&["\\[:space:]"])
.pipe_in("z\n")
.fails_with_code(1)
.no_output();
// `-F` treats the text literally, so no diagnostic.
let (_s, mut c) = ucmd();
c.args(&["-F", "[:space:]"])
.pipe_in("x\n")
.fails_with_code(1)
.no_output();
}
#[test]
fn reversed_range_uses_gnu_wording() {
// A range like `[b-a]` is an error; GNU prints the bare POSIX diagnostic
// "Invalid range end" (not oniguruma's phrasing) and exits 2.
// No piped input: the pattern is rejected before stdin is read.
for args in [&["[b-a]"][..], &["-E", "[b-a]"][..]] {
let (_s, mut c) = ucmd();
c.args(args)
.fails_with_code(2)
.stderr_is("grep: Invalid range end\n");
}
}
#[test]
fn pcre_backtracking_limit_does_not_abort() {
// A pathological PCRE pattern can exceed oniguruma's retry limit. GNU
// grep reports this and exits 2 (it must not crash); stdout stays empty.
let (_s, mut c) = ucmd();
c.args(&["-P", "((a+)*)+$"])
.pipe_in("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab\n")
.fails_with_code(2)
.stdout_is("")
.stderr_contains("backtracking limit");
}
#[test]
fn invalid_backreference_uses_gnu_wording() {
// A back-reference to a non-existent group is worded differently by GNU
// depending on the engine: PCRE (-P) vs gnulib regex (BRE/ERE).
// No piped input: these patterns are rejected before stdin is read.
let (_s, mut c) = ucmd();
c.args(&["-P", r"(.)\2"])
.fails_with_code(2)
.stderr_is("grep: reference to non-existent subpattern\n");
for args in [&["-E", r"(.)\2"][..], &[r"\(.\)\2"][..]] {
let (_s, mut c) = ucmd();
c.args(args)
.fails_with_code(2)
.stderr_is("grep: Invalid back reference\n");
}
// A valid back-reference with -Pw / -Px must still match.
for flag in ["-Pw", "-Px"] {
let (_s, mut c) = ucmd();
c.args(&[flag, r"(.)\1"])
.pipe_in("aa\n")
.succeeds()
.stdout_is("aa\n");
}
}
#[test]
fn grep_colors_mt_and_grep_color_deprecation() {
// GREP_COLORS `mt` sets the match color (both selected and context).
let (_s, mut c) = ucmd();
c.args(&["--color=always", "."])
.env("GREP_COLORS", "mt=36")
.pipe_in("x\n")
.succeeds()
.stdout_is("\u{1b}[36m\u{1b}[Kx\u{1b}[m\u{1b}[K\n");
// GREP_COLOR is deprecated: it still sets the match color, but emits a
// warning when color is actually produced.
let (_s, mut c) = ucmd();
c.args(&["--color=always", "."])
.env("GREP_COLOR", "36")
.pipe_in("x\n")
.succeeds()
.stdout_is("\u{1b}[36m\u{1b}[Kx\u{1b}[m\u{1b}[K\n")
.stderr_is("grep: warning: GREP_COLOR='36' is deprecated; use GREP_COLORS='mt=36'\n");
// No warning when color output is disabled.
let (_s, mut c) = ucmd();
c.args(&["--color=never", "."])
.env("GREP_COLOR", "36")
.pipe_in("x\n")
.succeeds()
.stdout_is("x\n")
.no_stderr();
}
#[test]
fn fixed_string_is_literal() {
// Metacharacters are not interpreted.