mirror of
https://github.com/uutils/grep.git
synced 2026-06-10 16:15:11 -07:00
perf: buffer-at-a-time search for literal patterns
Literal searches were ~50-70x slower than GNU grep because every line paid per-line costs (terminator scan, NUL scan, dispatch) even when a buffer held no match. Add a buffer-at-a-time driver that scans whole chunks with a substring searcher and only locates line boundaries around the matches it finds; a chunk with no match costs a single vectorized sweep and no per-line work. The driver activates only for plain ASCII literal patterns (case sensitive, no metacharacters) in the simpler output modes: -c, -l, -L, -q, and plain line printing with -n/-b/filename/-m. Anything needing match positions, context, inversion, color, or special binary handling falls back to the unchanged line-at-a-time path. Output stays byte-identical to that path, including binary/invalid-UTF-8 behavior. - line_buffer: read_chunk() yields the largest span of complete lines. - matcher: expose per-pattern memmem searchers when every pattern is a plain literal (plain_literal()). - searcher: eligible_for_fast_path(), fast_locate(), fast_print(). All scanning rides on the memchr crate (SIMD memchr/memrchr/memmem). Unit tests for read_chunk and plain_literal; integration tests for prefixes, -m, and multi-chunk line-number correctness. Benchmarks (31 MB corpus) vs prior release: -F (no match): 232ms -> 15ms (15.9x; now faster than GNU) -c literal: 229ms -> 15ms (15.2x) plain print: 248ms -> 18ms (13.5x) Regex and -i paths are unchanged (still the line-at-a-time engine).
This commit is contained in:
+1
-1
@@ -22,7 +22,7 @@ use std::io::{IsTerminal as _, Read};
|
||||
use std::path::Path;
|
||||
use uucore::error::{FromIo, UResult, USimpleError};
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
#[doc(hidden)]
|
||||
pub enum RegexMode {
|
||||
Fixed,
|
||||
|
||||
+190
-1
@@ -3,7 +3,7 @@
|
||||
// For the full copyright and license information, please view the LICENSE
|
||||
// file that was distributed with this source code.
|
||||
|
||||
use memchr::memchr;
|
||||
use memchr::{memchr, memrchr};
|
||||
use std::fs::File;
|
||||
use std::io::{self, Read as _};
|
||||
|
||||
@@ -111,4 +111,193 @@ impl LineBuffer {
|
||||
self.end += n;
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the next run of *complete* lines as a single slice.
|
||||
///
|
||||
/// Returns `Ok(None)` at end of input. Otherwise returns `Ok(Some((chunk,
|
||||
/// chunk_start)))`, where `chunk` spans one or more whole lines (each ending
|
||||
/// in the terminator) and `chunk_start` is the absolute byte offset of the
|
||||
/// first byte of the chunk. The only exception is a final line lacking a
|
||||
/// terminator, which is returned on its own as the last chunk.
|
||||
///
|
||||
/// This hands back as much buffered data as ends on a line boundary, so a
|
||||
/// caller can scan many lines with one pass instead of line by line.
|
||||
pub fn read_chunk(&mut self, file: &mut File) -> io::Result<Option<(&[u8], u64)>> {
|
||||
loop {
|
||||
// Hand back everything up to and including the last terminator.
|
||||
if self.end > self.beg
|
||||
&& let Some(off) = memrchr(self.line_terminator, &self.buffer[self.beg..self.end])
|
||||
{
|
||||
let beg = self.beg;
|
||||
let lim = self.beg + off + 1;
|
||||
let chunk_start = self.next_line_start;
|
||||
self.next_line_start += (lim - beg) as u64;
|
||||
self.beg = lim;
|
||||
self.scan = lim;
|
||||
return Ok(Some((&self.buffer[beg..lim], chunk_start)));
|
||||
}
|
||||
|
||||
// No whole line buffered. At EOF, flush any unterminated remainder.
|
||||
if self.eof {
|
||||
if self.beg == self.end {
|
||||
return Ok(None);
|
||||
}
|
||||
let beg = self.beg;
|
||||
let chunk_start = self.next_line_start;
|
||||
self.next_line_start += (self.end - beg) as u64;
|
||||
self.beg = self.end;
|
||||
self.scan = self.end;
|
||||
return Ok(Some((&self.buffer[beg..self.end], chunk_start)));
|
||||
}
|
||||
|
||||
// Slide the partial tail to the front to maximize room for reading.
|
||||
if self.beg > 0 {
|
||||
self.buffer.copy_within(self.beg..self.end, 0);
|
||||
self.end -= self.beg;
|
||||
self.beg = 0;
|
||||
self.scan = 0;
|
||||
}
|
||||
if self.end == self.buffer.len() {
|
||||
// A single line is longer than the whole buffer; grow it.
|
||||
self.buffer.resize(self.buffer.len() * 2, 0);
|
||||
}
|
||||
|
||||
let n = loop {
|
||||
match file.read(&mut self.buffer[self.end..]) {
|
||||
Ok(n) => break n,
|
||||
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
};
|
||||
if n == 0 {
|
||||
self.eof = true;
|
||||
} else {
|
||||
self.end += n;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::{Seek as _, SeekFrom, Write as _};
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
|
||||
static COUNTER: AtomicU32 = AtomicU32::new(0);
|
||||
|
||||
/// A temp file pre-loaded with `content`, rewound to the start, and removed
|
||||
/// from disk when dropped.
|
||||
struct TempInput {
|
||||
file: File,
|
||||
path: std::path::PathBuf,
|
||||
}
|
||||
|
||||
impl Drop for TempInput {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
fn temp_input(content: &[u8]) -> TempInput {
|
||||
let mut path = std::env::temp_dir();
|
||||
let n = COUNTER.fetch_add(1, Ordering::Relaxed);
|
||||
path.push(format!("uu_grep_lb_{}_{n}.tmp", std::process::id()));
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(&path)
|
||||
.unwrap();
|
||||
file.write_all(content).unwrap();
|
||||
file.seek(SeekFrom::Start(0)).unwrap();
|
||||
TempInput { file, path }
|
||||
}
|
||||
|
||||
/// Drain `read_chunk` into a list of (owned bytes, start offset) pairs.
|
||||
fn chunks(term: u8, content: &[u8]) -> Vec<(Vec<u8>, u64)> {
|
||||
let mut lb = LineBuffer::new(term);
|
||||
let mut input = temp_input(content);
|
||||
let mut out = Vec::new();
|
||||
while let Some((chunk, start)) = lb.read_chunk(&mut input.file).unwrap() {
|
||||
out.push((chunk.to_vec(), start));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_yields_nothing() {
|
||||
assert!(chunks(b'\n', b"").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whole_complete_lines_come_back_as_one_chunk() {
|
||||
// Small input arrives in a single read, so everything up to the final
|
||||
// terminator is one chunk starting at offset 0.
|
||||
assert_eq!(
|
||||
chunks(b'\n', b"a\nbb\nccc\n"),
|
||||
vec![(b"a\nbb\nccc\n".to_vec(), 0)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unterminated_tail_is_a_final_chunk_with_its_own_offset() {
|
||||
// "a\n" is the complete-line chunk; "bb" is flushed at EOF at offset 2.
|
||||
assert_eq!(
|
||||
chunks(b'\n', b"a\nbb"),
|
||||
vec![(b"a\n".to_vec(), 0), (b"bb".to_vec(), 2)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_without_any_terminator_is_one_chunk() {
|
||||
assert_eq!(chunks(b'\n', b"abc"), vec![(b"abc".to_vec(), 0)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn honors_a_custom_terminator() {
|
||||
assert_eq!(
|
||||
chunks(b'\0', b"a\0bb\0c"),
|
||||
vec![(b"a\0bb\0".to_vec(), 0), (b"c".to_vec(), 5)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reassembles_input_larger_than_the_buffer() {
|
||||
// Force many reads and at least one chunk boundary mid-file.
|
||||
let mut content = Vec::new();
|
||||
for i in 0..50_000u32 {
|
||||
content.extend_from_slice(format!("line number {i}\n").as_bytes());
|
||||
}
|
||||
assert!(content.len() > 128 * 1024);
|
||||
|
||||
let got = chunks(b'\n', &content);
|
||||
assert!(got.len() > 1, "expected multiple chunks, got {}", got.len());
|
||||
|
||||
// Chunks must tile the input exactly, contiguously, each ending on a
|
||||
// line boundary (the input ends with a terminator).
|
||||
let mut expected_start = 0u64;
|
||||
let mut joined = Vec::new();
|
||||
for (bytes, start) in &got {
|
||||
assert_eq!(*start, expected_start);
|
||||
assert_eq!(*bytes.last().unwrap(), b'\n');
|
||||
expected_start += bytes.len() as u64;
|
||||
joined.extend_from_slice(bytes);
|
||||
}
|
||||
assert_eq!(joined, content);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grows_to_hold_a_single_overlong_line() {
|
||||
// One line far bigger than the initial 128 KiB buffer, then a short one.
|
||||
let mut content = vec![b'x'; 300 * 1024];
|
||||
content.push(b'\n');
|
||||
content.extend_from_slice(b"tail\n");
|
||||
|
||||
let got = chunks(b'\n', &content);
|
||||
let joined: Vec<u8> = got.iter().flat_map(|(b, _)| b.clone()).collect();
|
||||
assert_eq!(joined, content);
|
||||
assert_eq!(got[0].1, 0);
|
||||
}
|
||||
}
|
||||
|
||||
+98
-1
@@ -4,6 +4,7 @@
|
||||
// file that was distributed with this source code.
|
||||
|
||||
use crate::{Config, RegexMode};
|
||||
use memchr::memmem;
|
||||
use onig::{
|
||||
EncodedBytes, Regex, RegexOptions, Region, SearchOptions, Syntax, SyntaxBehavior,
|
||||
SyntaxOperator,
|
||||
@@ -14,6 +15,12 @@ use uucore::error::{UResult, USimpleError};
|
||||
pub struct Matcher<'a> {
|
||||
config: &'a Config<'a>,
|
||||
patterns: Vec<CompiledPattern>,
|
||||
/// One substring searcher per pattern, present only when *every* pattern is
|
||||
/// a plain literal that a raw byte search resolves exactly (see
|
||||
/// [`plain_literal`]). When set, a caller can decide a line matches by
|
||||
/// looking for any of these needles, bypassing the regex engine entirely.
|
||||
/// `None` as soon as a single pattern needs real regex evaluation.
|
||||
literal_searchers: Option<Vec<memmem::Finder<'static>>>,
|
||||
}
|
||||
|
||||
impl<'a> Matcher<'a> {
|
||||
@@ -22,7 +29,32 @@ impl<'a> Matcher<'a> {
|
||||
for raw in config.patterns {
|
||||
patterns.push(CompiledPattern::compile(raw, config)?);
|
||||
}
|
||||
Ok(Self { config, patterns })
|
||||
|
||||
// If we can reduce the whole pattern set to literal needles, keep a
|
||||
// searcher for each so the driver can take a bulk substring-scan path.
|
||||
let needles: Option<Vec<Vec<u8>>> = config
|
||||
.patterns
|
||||
.iter()
|
||||
.map(|p| plain_literal(p, config.ignore_case, config.regex_mode))
|
||||
.collect();
|
||||
let literal_searchers = needles.filter(|n| !n.is_empty()).map(|n| {
|
||||
n.iter()
|
||||
.map(|w| memmem::Finder::new(w).into_owned())
|
||||
.collect()
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
patterns,
|
||||
literal_searchers,
|
||||
})
|
||||
}
|
||||
|
||||
/// Per-pattern substring searchers, present only when the pattern set is a
|
||||
/// pure set of literals (no regex needed). Used by the searcher to scan a
|
||||
/// whole buffer at once instead of testing line by line.
|
||||
pub fn literal_searchers(&self) -> Option<&[memmem::Finder<'static>]> {
|
||||
self.literal_searchers.as_deref()
|
||||
}
|
||||
|
||||
/// Decide whether `line` matches and return the positions to highlight.
|
||||
@@ -194,6 +226,25 @@ impl Cursor<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the literal bytes of `pattern` when a raw byte-for-byte substring
|
||||
/// search is *exactly* equivalent to matching it, otherwise `None`.
|
||||
///
|
||||
/// We accept only ASCII, case-sensitive needles. That keeps the byte search in
|
||||
/// agreement with the regex engine on every possible input, including bytes that
|
||||
/// are not valid UTF-8: an ASCII byte can never be part of a multi-byte sequence,
|
||||
/// so its presence is unambiguous. In the regex modes we also require that no
|
||||
/// byte could ever act as a metacharacter; under `-F` the text is literal as-is.
|
||||
fn plain_literal(pattern: &str, ignore_case: bool, mode: RegexMode) -> Option<Vec<u8>> {
|
||||
if ignore_case || pattern.is_empty() || !pattern.is_ascii() {
|
||||
return None;
|
||||
}
|
||||
// Every byte that carries special meaning in any of our regex syntaxes.
|
||||
// A needle without these reads the same as a literal in Basic/Extended/Perl.
|
||||
const SPECIAL: &[u8] = b".*[]^$\\+?{}()|";
|
||||
let plain = mode == RegexMode::Fixed || !pattern.bytes().any(|b| SPECIAL.contains(&b));
|
||||
plain.then(|| pattern.as_bytes().to_vec())
|
||||
}
|
||||
|
||||
struct CompiledPattern {
|
||||
/// Default semantics. It's decently fast and used for searching.
|
||||
leftmost: Regex,
|
||||
@@ -289,3 +340,49 @@ impl CompiledPattern {
|
||||
.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::plain_literal;
|
||||
use crate::RegexMode;
|
||||
|
||||
fn lit(p: &str, ic: bool, mode: RegexMode) -> Option<Vec<u8>> {
|
||||
plain_literal(p, ic, mode)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixed_mode_takes_any_ascii_verbatim() {
|
||||
// Under -F every byte is literal, even regex metacharacters.
|
||||
assert_eq!(lit("abc", false, RegexMode::Fixed), Some(b"abc".to_vec()));
|
||||
assert_eq!(lit("a.*b", false, RegexMode::Fixed), Some(b"a.*b".to_vec()));
|
||||
assert_eq!(lit("a+b", false, RegexMode::Fixed), Some(b"a+b".to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regex_modes_accept_metacharacter_free_literals() {
|
||||
for mode in [RegexMode::Basic, RegexMode::Extended, RegexMode::Perl] {
|
||||
assert_eq!(lit("ing", false, mode), Some(b"ing".to_vec()));
|
||||
assert_eq!(lit("Hello123", false, mode), Some(b"Hello123".to_vec()));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regex_modes_reject_anything_with_a_metacharacter() {
|
||||
for mode in [RegexMode::Basic, RegexMode::Extended, RegexMode::Perl] {
|
||||
for p in [
|
||||
"a.b", "a*", "[ab]", "^a", "a$", "a\\b", "a+", "a?", "(a)", "a|b", "a{2}",
|
||||
] {
|
||||
assert_eq!(lit(p, false, mode), None, "pattern {p:?} in {mode:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_case_insensitive_and_non_ascii() {
|
||||
assert_eq!(lit("", false, RegexMode::Fixed), None);
|
||||
assert_eq!(lit("abc", true, RegexMode::Fixed), None); // -i
|
||||
assert_eq!(lit("abc", true, RegexMode::Basic), None);
|
||||
assert_eq!(lit("café", false, RegexMode::Fixed), None); // non-ASCII
|
||||
assert_eq!(lit("naïve", false, RegexMode::Basic), None);
|
||||
}
|
||||
}
|
||||
|
||||
+239
-1
@@ -8,7 +8,8 @@ use crate::line_buffer::LineBuffer;
|
||||
use crate::matcher::Matcher;
|
||||
use crate::output::OutputWriter;
|
||||
use crate::{BinaryMode, Config, DeviceMode, DirectoryMode};
|
||||
use memchr::memchr;
|
||||
use memchr::memmem::Finder;
|
||||
use memchr::{memchr, memchr_iter, memrchr};
|
||||
use std::ffi::OsStr;
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
@@ -248,12 +249,221 @@ impl<'a> Searcher<'a> {
|
||||
self.binary_notice_enabled && self.session_binary_detected && self.session_any_match()
|
||||
}
|
||||
|
||||
/// Whether the current configuration can use the buffer-at-a-time fast
|
||||
/// path. It applies only to pure-literal patterns and the simpler output
|
||||
/// modes — anything needing match positions, context, inversion, or special
|
||||
/// binary handling falls back to the line-at-a-time [`Self::session_run`].
|
||||
fn eligible_for_fast_path(&self) -> bool {
|
||||
// On Windows the line-at-a-time path strips a trailing CR before
|
||||
// matching; the fast path mirrors that only for printed output, so a
|
||||
// literal needle still behaves the same. Nothing else differs.
|
||||
self.matcher.literal_searchers().is_some()
|
||||
&& !self.config.invert_match
|
||||
&& !self.config.word_regexp
|
||||
&& !self.config.line_regexp
|
||||
&& !self.config.only_matching
|
||||
&& !self.config.use_color
|
||||
// `has_context` also covers `-C 0`, which still emits `--` separators.
|
||||
&& !self.config.has_context
|
||||
&& !self.config.null_data
|
||||
&& self.config.binary_mode != BinaryMode::WithoutMatch
|
||||
}
|
||||
|
||||
/// Buffer-at-a-time driver for literal patterns. Instead of testing every
|
||||
/// line, it scans whole chunks with a substring searcher and only locates
|
||||
/// line boundaries around the matches it finds.
|
||||
fn session_run_fast(
|
||||
&mut self,
|
||||
lb: &mut LineBuffer,
|
||||
path: &Path,
|
||||
reader: &mut File,
|
||||
) -> io::Result<bool> {
|
||||
lb.reset();
|
||||
if self.config.quiet
|
||||
|| self.config.files_with_matches
|
||||
|| self.config.files_without_match
|
||||
|| self.config.count
|
||||
{
|
||||
self.fast_locate(lb, path, reader)
|
||||
} else {
|
||||
self.fast_print(lb, path, reader)
|
||||
}
|
||||
}
|
||||
|
||||
/// Fast path for modes that only need to know *whether* / *how many* lines
|
||||
/// match: `-c`, `-l`, `-L`, `-q`. No per-line rendering, so no line numbers,
|
||||
/// byte offsets, or binary bookkeeping are required (the count of matching
|
||||
/// lines is unaffected by binary detection, and `-l`/`-L`/`-q` list files
|
||||
/// regardless).
|
||||
fn fast_locate(
|
||||
&mut self,
|
||||
lb: &mut LineBuffer,
|
||||
path: &Path,
|
||||
reader: &mut File,
|
||||
) -> io::Result<bool> {
|
||||
let finders = self
|
||||
.matcher
|
||||
.literal_searchers()
|
||||
.expect("eligibility guarantees literal searchers");
|
||||
let max = self.config.max_count;
|
||||
// Existence is enough for these three; only `-c` needs the full tally.
|
||||
let stop_at_first =
|
||||
self.config.quiet || self.config.files_with_matches || self.config.files_without_match;
|
||||
|
||||
let mut count: u64 = 0;
|
||||
let mut matched = false;
|
||||
'outer: while let Some((chunk, _)) = lb.read_chunk(reader)? {
|
||||
let mut p = 0;
|
||||
while p < chunk.len() {
|
||||
let Some(rel) = leftmost_match(finders, &chunk[p..]) else {
|
||||
break;
|
||||
};
|
||||
if max.is_some_and(|mx| count >= mx) {
|
||||
break 'outer;
|
||||
}
|
||||
let (_, line_end) = line_bounds(chunk, p + rel);
|
||||
count += 1;
|
||||
matched = true;
|
||||
if stop_at_first {
|
||||
break 'outer;
|
||||
}
|
||||
// Each line counts once: resume past this line's terminator.
|
||||
p = line_end + 1;
|
||||
}
|
||||
}
|
||||
|
||||
// `-l`/`-L` take precedence over `-c`, matching the line-at-a-time path.
|
||||
if self.config.quiet {
|
||||
// Exit status only.
|
||||
} else if self.config.files_with_matches {
|
||||
if matched {
|
||||
self.writer.write_filename(path)?;
|
||||
}
|
||||
} else if self.config.files_without_match {
|
||||
if !matched {
|
||||
self.writer.write_filename(path)?;
|
||||
}
|
||||
} else if self.config.count {
|
||||
self.writer.write_count(count, path)?;
|
||||
}
|
||||
Ok(matched)
|
||||
}
|
||||
|
||||
/// Fast path that prints whole matching lines (optionally with `-n`, `-b`,
|
||||
/// filename prefixes, `-m`). Binary files are detected per chunk and reported
|
||||
/// with the usual notice instead of dumping their lines.
|
||||
fn fast_print(
|
||||
&mut self,
|
||||
lb: &mut LineBuffer,
|
||||
path: &Path,
|
||||
reader: &mut File,
|
||||
) -> io::Result<bool> {
|
||||
let finders = self
|
||||
.matcher
|
||||
.literal_searchers()
|
||||
.expect("eligibility guarantees literal searchers");
|
||||
let max = self.config.max_count;
|
||||
let want_lineno = self.config.line_number;
|
||||
let detect_binary = self.config.binary_mode != BinaryMode::Text;
|
||||
let notice_enabled = self.binary_notice_enabled;
|
||||
|
||||
let mut count: u64 = 0;
|
||||
let mut matched = false;
|
||||
let mut binary = false;
|
||||
// Number of terminators in all previously consumed chunks (for `-n`).
|
||||
let mut base_lines: u64 = 0;
|
||||
|
||||
'outer: while let Some((chunk, chunk_off)) = lb.read_chunk(reader)? {
|
||||
let mut p = 0;
|
||||
// NUL scanned up to here; terminators counted up to `nl_cursor`.
|
||||
let mut nul_scanned = 0;
|
||||
let mut nl_cursor = 0;
|
||||
let mut nl_before = 0u64;
|
||||
|
||||
while p < chunk.len() {
|
||||
let Some(rel) = leftmost_match(finders, &chunk[p..]) else {
|
||||
break;
|
||||
};
|
||||
if max.is_some_and(|mx| count >= mx) {
|
||||
break 'outer;
|
||||
}
|
||||
let (line_beg, line_end) = line_bounds(chunk, p + rel);
|
||||
|
||||
// A NUL anywhere up to this line marks the file binary, as does
|
||||
// an invalid-UTF-8 matching line.
|
||||
if detect_binary && !binary {
|
||||
if memchr(0, &chunk[nul_scanned..line_end]).is_some() {
|
||||
binary = true;
|
||||
}
|
||||
nul_scanned = line_end;
|
||||
}
|
||||
|
||||
let line = &chunk[line_beg..line_end];
|
||||
#[cfg(windows)]
|
||||
let line = if self.config.strip_cr && line.last() == Some(&b'\r') {
|
||||
&line[..line.len() - 1]
|
||||
} else {
|
||||
line
|
||||
};
|
||||
|
||||
if detect_binary && !binary && std::str::from_utf8(line).is_err() {
|
||||
binary = true;
|
||||
}
|
||||
|
||||
if binary {
|
||||
// First match in a binary file: stop and emit the notice
|
||||
// once at the end instead of dumping the line.
|
||||
matched = true;
|
||||
break 'outer;
|
||||
}
|
||||
|
||||
let line_number = if want_lineno {
|
||||
nl_before += count_terminators(&chunk[nl_cursor..line_beg]);
|
||||
nl_cursor = line_beg;
|
||||
base_lines + nl_before + 1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
self.writer.write_line(
|
||||
&LineView {
|
||||
line,
|
||||
line_number,
|
||||
byte_offset: chunk_off + line_beg as u64,
|
||||
is_match: true,
|
||||
match_positions: &[],
|
||||
},
|
||||
path,
|
||||
)?;
|
||||
count += 1;
|
||||
matched = true;
|
||||
p = line_end + 1;
|
||||
}
|
||||
|
||||
// Carry NUL detection and the line tally across the chunk boundary.
|
||||
if detect_binary && !binary && memchr(0, &chunk[nul_scanned..]).is_some() {
|
||||
binary = true;
|
||||
}
|
||||
if want_lineno {
|
||||
base_lines += nl_before + count_terminators(&chunk[nl_cursor..]);
|
||||
}
|
||||
}
|
||||
|
||||
if binary && notice_enabled && matched {
|
||||
self.writer.report_binary_match(path);
|
||||
}
|
||||
Ok(matched)
|
||||
}
|
||||
|
||||
fn session_run(
|
||||
&mut self,
|
||||
lb: &mut LineBuffer,
|
||||
path: &Path,
|
||||
reader: &mut File,
|
||||
) -> io::Result<bool> {
|
||||
if self.eligible_for_fast_path() {
|
||||
return self.session_run_fast(lb, path, reader);
|
||||
}
|
||||
|
||||
// Reset all session (per-file) state.
|
||||
self.session_context_buf.clear();
|
||||
self.session_match_count = 0;
|
||||
@@ -470,3 +680,31 @@ impl<'a> Searcher<'a> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Offset of the earliest occurrence of any needle in `hay`, or `None`.
|
||||
fn leftmost_match(finders: &[Finder<'static>], hay: &[u8]) -> Option<usize> {
|
||||
let mut best: Option<usize> = None;
|
||||
for finder in finders {
|
||||
if let Some(pos) = finder.find(hay) {
|
||||
best = Some(best.map_or(pos, |b| b.min(pos)));
|
||||
if best == Some(0) {
|
||||
break; // Can't start any earlier.
|
||||
}
|
||||
}
|
||||
}
|
||||
best
|
||||
}
|
||||
|
||||
/// Count line terminators in `bytes`.
|
||||
fn count_terminators(bytes: &[u8]) -> u64 {
|
||||
memchr_iter(b'\n', bytes).count() as u64
|
||||
}
|
||||
|
||||
/// Byte range `[start, end)` of the line containing `pos` in `buf`, excluding
|
||||
/// the trailing terminator. `start` follows the previous terminator (or 0);
|
||||
/// `end` is the next terminator (or end of buffer).
|
||||
fn line_bounds(buf: &[u8], pos: usize) -> (usize, usize) {
|
||||
let start = memrchr(b'\n', &buf[..pos]).map_or(0, |i| i + 1);
|
||||
let end = memchr(b'\n', &buf[pos..]).map_or(buf.len(), |i| pos + i);
|
||||
(start, end)
|
||||
}
|
||||
|
||||
@@ -1272,3 +1272,70 @@ fn repeated_options_are_accepted() {
|
||||
.succeeds()
|
||||
.stdout_only("a\nb\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn literal_buffer_path_prefixes_and_max() {
|
||||
// Plain literals are served by the buffer-at-a-time engine; the line/byte
|
||||
// prefixes and -m must still be byte-identical to the line-at-a-time path.
|
||||
|
||||
// -n and -b together: "lineno:byteoffset:line".
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["-nb", "foo"])
|
||||
.pipe_in("foo\nbar\nfoobar\n")
|
||||
.succeeds()
|
||||
.stdout_only("1:0:foo\n3:8:foobar\n");
|
||||
|
||||
// A line matched more than once is still emitted once.
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["-c", "oo"])
|
||||
.pipe_in("oooo\nbar\noo\n")
|
||||
.succeeds()
|
||||
.stdout_only("2\n");
|
||||
|
||||
// -m caps printed matches.
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["-m", "2", "x"])
|
||||
.pipe_in("x\ny\nx\nz\nx\n")
|
||||
.succeeds()
|
||||
.stdout_only("x\nx\n");
|
||||
|
||||
// Final line without a trailing terminator still matches and is printed
|
||||
// with an added newline.
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["foo"])
|
||||
.pipe_in("bar\nfoo")
|
||||
.succeeds()
|
||||
.stdout_only("foo\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn literal_buffer_path_spans_many_chunks() {
|
||||
// Build an input far larger than the read buffer so the buffer-at-a-time
|
||||
// engine crosses several chunk boundaries, and check that line numbers and
|
||||
// counts stay correct across them.
|
||||
let mut input = String::new();
|
||||
let mut expected_n = String::new();
|
||||
let mut count = 0u32;
|
||||
for i in 1..=100_000u32 {
|
||||
if i % 7 == 0 {
|
||||
input.push_str("needle\n");
|
||||
expected_n.push_str(&format!("{i}:needle\n"));
|
||||
count += 1;
|
||||
} else {
|
||||
input.push_str("some filler text\n");
|
||||
}
|
||||
}
|
||||
assert!(input.len() > 512 * 1024, "input must exceed several chunks");
|
||||
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["-c", "needle"])
|
||||
.pipe_in(input.clone())
|
||||
.succeeds()
|
||||
.stdout_only(format!("{count}\n"));
|
||||
|
||||
let (_s, mut c) = ucmd();
|
||||
c.args(&["-n", "needle"])
|
||||
.pipe_in(input)
|
||||
.succeeds()
|
||||
.stdout_only(expected_n);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user