From 36cc88e1590bf4fb2c329edd73a67c93916f55ff Mon Sep 17 00:00:00 2001 From: Tavian Barnes Date: Wed, 23 Feb 2022 20:48:39 -0500 Subject: [PATCH] find/matchers: Implement POSIX-compliant globs The glob crate exposes .gitignore style globs with recursive (**) matching and other features. find should implement globs exactly as specified by POSIX. Do this by translating them to POSIX Basic Regular Expressions. Fixes #140. --- Cargo.lock | 1 - Cargo.toml | 1 - src/find/main.rs | 3 - src/find/matchers/glob.rs | 238 +++++++++++++++++++++++++++++++++++++ src/find/matchers/lname.rs | 29 ++--- src/find/matchers/mod.rs | 5 +- src/find/matchers/name.rs | 52 ++------ src/lib.rs | 7 -- 8 files changed, 261 insertions(+), 75 deletions(-) create mode 100644 src/find/matchers/glob.rs diff --git a/Cargo.lock b/Cargo.lock index a3003c0..de68692 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -223,7 +223,6 @@ dependencies = [ "chrono", "clap", "filetime", - "glob", "once_cell", "onig", "predicates", diff --git a/Cargo.toml b/Cargo.toml index df9e9c2..96b72e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,6 @@ authors = ["uutils developers"] [dependencies] chrono = "0.4" clap = "2.34" -glob = "0.3" walkdir = "2.3" regex = "1.5" once_cell = "1.9" diff --git a/src/find/main.rs b/src/find/main.rs index 1b6b324..f3e7a3f 100644 --- a/src/find/main.rs +++ b/src/find/main.rs @@ -4,9 +4,6 @@ // license that can be found in the LICENSE file or at // https://opensource.org/licenses/MIT. -extern crate findutils; -extern crate glob; - fn main() { let args = std::env::args().collect::>(); let strs: Vec<&str> = args.iter().map(|s| s.as_ref()).collect(); diff --git a/src/find/matchers/glob.rs b/src/find/matchers/glob.rs new file mode 100644 index 0000000..5695d1d --- /dev/null +++ b/src/find/matchers/glob.rs @@ -0,0 +1,238 @@ +// Copyright 2022 Tavian Barnes +// +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +use onig::{self, Regex, RegexOptions, Syntax}; + +/// Parse a string as a POSIX Basic Regular Expression. +fn parse_bre(expr: &str, options: RegexOptions) -> Result { + let bre = Syntax::posix_basic(); + Regex::with_options(expr, bre.options() | options, bre) +} + +/// Push a literal character onto a regex, escaping it if necessary. +fn regex_push_literal(regex: &mut String, ch: char) { + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_03_03 + if matches!(ch, '.' | '[' | '\\' | '*' | '^' | '$') { + regex.push('\\'); + } + regex.push(ch); +} + +/// Extracts a bracket expression from a glob. +fn extract_bracket_expr(pattern: &str) -> Option<(String, &str)> { + // https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_13_01 + // + // If an open bracket introduces a bracket expression as in XBD RE Bracket Expression, + // except that the character ( '!' ) shall replace the + // character ( '^' ) in its role in a non-matching list in the regular expression notation, + // it shall introduce a pattern bracket expression. A bracket expression starting with an + // unquoted character produces unspecified results. Otherwise, '[' shall match + // the character itself. + // + // To check for valid bracket expressions, we scan for the closing bracket and + // attempt to parse that segment as a regex. If that fails, we treat the '[' + // literally. + + let mut expr = "[".to_string(); + + let mut chars = pattern.chars(); + let mut next = chars.next(); + + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_03_05 + // + // 3. A non-matching list expression begins with a ( '^' ) ... + // + // (but in a glob, '!' is used instead of '^') + if next == Some('!') { + expr.push('^'); + next = chars.next(); + } + + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_03_05 + // + // 1. ... The ( ']' ) shall lose its special meaning and represent + // itself in a bracket expression if it occurs first in the list (after an initial + // ( '^' ), if any). + if next == Some(']') { + expr.push(']'); + next = chars.next(); + } + + while let Some(ch) = next { + expr.push(ch); + + match ch { + '[' => { + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_03_05 + // + // 4. A collating symbol is a collating element enclosed within bracket-period + // ( "[." and ".]" ) delimiters. ... + // + // 5. An equivalence class expression shall ... be expressed by enclosing any + // one of the collating elements in the equivalence class within bracket- + // equal ( "[=" and "=]" ) delimiters. + // + // 6. ... A character class expression is expressed as a character class name + // enclosed within bracket- ( "[:" and ":]" ) delimiters. + next = chars.next(); + if let Some(delim) = next { + expr.push(delim); + + if matches!(delim, '.' | '=' | ':') { + let rest = chars.as_str(); + let end = rest.find([delim, ']'])? + 2; + expr.push_str(&rest[..end]); + chars = rest[end..].chars(); + } + } + } + ']' => { + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_03_05 + // + // 1. ... The ( ']' ) shall ... terminate the bracket + // expression, unless it appears in a collating symbol (such as "[.].]" ) or is + // the ending for a collating symbol, equivalence class, + // or character class. + break; + } + _ => {} + } + + next = chars.next(); + } + + if parse_bre(&expr, RegexOptions::REGEX_OPTION_NONE).is_ok() { + Some((expr, chars.as_str())) + } else { + None + } +} + +/// Converts a POSIX glob into a POSIX Basic Regular Expression +fn glob_to_regex(pattern: &str) -> String { + let mut regex = String::new(); + + let mut chars = pattern.chars(); + while let Some(ch) = chars.next() { + // https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_13 + match ch { + '?' => regex.push('.'), + '*' => regex.push_str(".*"), + '\\' => { + if let Some(ch) = chars.next() { + regex_push_literal(&mut regex, ch); + } else { + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/fnmatch.html + // + // If pattern ends with an unescaped , fnmatch() shall return a + // non-zero value (indicating either no match or an error). + // + // Most implementations return FNM_NOMATCH in this case, so return a regex that + // never matches. + return "$.".to_string(); + } + } + '[' => { + if let Some((expr, rest)) = extract_bracket_expr(chars.as_str()) { + regex.push_str(&expr); + chars = rest.chars(); + } else { + regex_push_literal(&mut regex, ch); + } + } + _ => regex_push_literal(&mut regex, ch), + } + } + + regex +} + +/// An fnmatch()-style glob matcher. +pub struct Pattern { + regex: Regex, +} + +impl Pattern { + /// Parse an fnmatch()-style glob. + pub fn new(pattern: &str, caseless: bool) -> Self { + let options = if caseless { + RegexOptions::REGEX_OPTION_IGNORECASE + } else { + RegexOptions::REGEX_OPTION_NONE + }; + + // As long as glob_to_regex() is correct, this should never fail + let regex = parse_bre(&glob_to_regex(pattern), options).unwrap(); + Self { regex } + } + + /// Test if this patern matches a string. + pub fn matches(&self, string: &str) -> bool { + self.regex.is_match(string) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn literals() { + assert_eq!(glob_to_regex(r"foo.bar"), r"foo\.bar"); + } + + #[test] + fn regex_special() { + assert_eq!(glob_to_regex(r"^foo.bar$"), r"\^foo\.bar\$"); + } + + #[test] + fn wildcards() { + assert_eq!(glob_to_regex(r"foo?bar*baz"), r"foo.bar.*baz"); + } + + #[test] + fn escapes() { + assert_eq!(glob_to_regex(r"fo\o\?bar\*baz\\"), r"foo?bar\*baz\\"); + } + + #[test] + fn incomplete_escape() { + assert_eq!(glob_to_regex(r"foo\"), r"$.") + } + + #[test] + fn valid_brackets() { + assert_eq!(glob_to_regex(r"foo[bar][!baz]"), r"foo[bar][^baz]"); + } + + #[test] + fn complex_brackets() { + assert_eq!( + glob_to_regex(r"[!]!.*[\[.].][=]=][:space:]-]"), + r"[^]!.*[\[.].][=]=][:space:]-]" + ); + } + + #[test] + fn invalid_brackets() { + assert_eq!(glob_to_regex(r"foo[bar[!baz"), r"foo\[bar\[!baz"); + } + + #[test] + fn pattern_matches() { + assert!(Pattern::new(r"foo*bar", false).matches("foo--bar")); + + assert!(!Pattern::new(r"foo*bar", false).matches("bar--foo")); + } + + #[test] + fn caseless_matches() { + assert!(Pattern::new(r"foo*BAR", true).matches("FOO--bar")); + + assert!(!Pattern::new(r"foo*BAR", true).matches("BAR--foo")); + } +} diff --git a/src/find/matchers/lname.rs b/src/find/matchers/lname.rs index 40e78e9..58c130e 100644 --- a/src/find/matchers/lname.rs +++ b/src/find/matchers/lname.rs @@ -7,10 +7,9 @@ use std::io::{stderr, Write}; use std::path::PathBuf; -use glob::Pattern; -use glob::PatternError; use walkdir::DirEntry; +use super::glob::Pattern; use super::{Matcher, MatcherIO}; fn read_link_target(file_info: &DirEntry) -> Option { @@ -38,33 +37,19 @@ fn read_link_target(file_info: &DirEntry) -> Option { /// pattern. See `glob::Pattern` for details on the exact syntax. pub struct LinkNameMatcher { pattern: Pattern, - caseless: bool, } impl LinkNameMatcher { - pub fn new(pattern_string: &str, caseless: bool) -> Result { - let pattern = if caseless { - Pattern::new(&pattern_string.to_lowercase())? - } else { - Pattern::new(pattern_string)? - }; - - Ok(Self { - pattern, - caseless, - }) + pub fn new(pattern_string: &str, caseless: bool) -> LinkNameMatcher { + let pattern = Pattern::new(pattern_string, caseless); + Self { pattern } } } impl Matcher for LinkNameMatcher { fn matches(&self, file_info: &DirEntry, _: &mut MatcherIO) -> bool { if let Some(target) = read_link_target(file_info) { - let target = target.to_string_lossy(); - if self.caseless { - self.pattern.matches(&target.to_lowercase()) - } else { - self.pattern.matches(&target) - } + self.pattern.matches(&target.to_string_lossy()) } else { false } @@ -105,7 +90,7 @@ mod tests { create_file_link(); let link_f = get_dir_entry_for("test_data/links", "link-f"); - let matcher = LinkNameMatcher::new("ab?bc", false).unwrap(); + let matcher = LinkNameMatcher::new("ab?bc", false); let deps = FakeDependencies::new(); assert!(matcher.matches(&link_f, &mut deps.new_matcher_io())); } @@ -115,7 +100,7 @@ mod tests { create_file_link(); let link_f = get_dir_entry_for("test_data/links", "link-f"); - let matcher = LinkNameMatcher::new("AbB?c", true).unwrap(); + let matcher = LinkNameMatcher::new("AbB?c", true); let deps = FakeDependencies::new(); assert!(matcher.matches(&link_f, &mut deps.new_matcher_io())); } diff --git a/src/find/matchers/mod.rs b/src/find/matchers/mod.rs index 431f153..803552a 100644 --- a/src/find/matchers/mod.rs +++ b/src/find/matchers/mod.rs @@ -7,6 +7,7 @@ mod delete; mod empty; pub mod exec; +mod glob; mod lname; mod logical_matchers; mod name; @@ -286,14 +287,14 @@ fn build_matcher_tree( return Err(From::from(format!("missing argument to {}", args[i]))); } i += 1; - Some(LinkNameMatcher::new(args[i], args[i - 1].starts_with("-i"))?.into_box()) + Some(LinkNameMatcher::new(args[i], args[i - 1].starts_with("-i")).into_box()) } "-name" | "-iname" => { if i >= args.len() - 1 { return Err(From::from(format!("missing argument to {}", args[i]))); } i += 1; - Some(NameMatcher::new(args[i], args[i - 1].starts_with("-i"))?.into_box()) + Some(NameMatcher::new(args[i], args[i - 1].starts_with("-i")).into_box()) } "-regextype" => { if i >= args.len() - 1 { diff --git a/src/find/matchers/name.rs b/src/find/matchers/name.rs index fc336d7..94d8eb9 100644 --- a/src/find/matchers/name.rs +++ b/src/find/matchers/name.rs @@ -4,42 +4,28 @@ // license that can be found in the LICENSE file or at // https://opensource.org/licenses/MIT. -use glob::Pattern; -use glob::PatternError; use walkdir::DirEntry; +use super::glob::Pattern; use super::{Matcher, MatcherIO}; /// This matcher makes a comparison of the name against a shell wildcard /// pattern. See `glob::Pattern` for details on the exact syntax. pub struct NameMatcher { pattern: Pattern, - caseless: bool, } impl NameMatcher { - pub fn new(pattern_string: &str, caseless: bool) -> Result { - let pattern = if caseless { - Pattern::new(&pattern_string.to_lowercase())? - } else { - Pattern::new(pattern_string)? - }; - - Ok(Self { - pattern, - caseless, - }) + pub fn new(pattern_string: &str, caseless: bool) -> Self { + let pattern = Pattern::new(pattern_string, caseless); + Self { pattern } } } impl Matcher for NameMatcher { fn matches(&self, file_info: &DirEntry, _: &mut MatcherIO) -> bool { let name = file_info.file_name().to_string_lossy(); - if self.caseless { - self.pattern.matches(&name.to_lowercase()) - } else { - self.pattern.matches(&name) - } + self.pattern.matches(&name) } } @@ -76,7 +62,7 @@ mod tests { #[test] fn matching_with_wrong_case_returns_false() { let abbbc = get_dir_entry_for("test_data/simple", "abbbc"); - let matcher = NameMatcher::new("A*C", false).unwrap(); + let matcher = NameMatcher::new("A*C", false); let deps = FakeDependencies::new(); assert!(!matcher.matches(&abbbc, &mut deps.new_matcher_io())); } @@ -84,7 +70,7 @@ mod tests { #[test] fn matching_with_right_case_returns_true() { let abbbc = get_dir_entry_for("test_data/simple", "abbbc"); - let matcher = NameMatcher::new("abb?c", false).unwrap(); + let matcher = NameMatcher::new("abb?c", false); let deps = FakeDependencies::new(); assert!(matcher.matches(&abbbc, &mut deps.new_matcher_io())); } @@ -92,7 +78,7 @@ mod tests { #[test] fn not_matching_returns_false() { let abbbc = get_dir_entry_for("test_data/simple", "abbbc"); - let matcher = NameMatcher::new("shouldn't match", false).unwrap(); + let matcher = NameMatcher::new("shouldn't match", false); let deps = FakeDependencies::new(); assert!(!matcher.matches(&abbbc, &mut deps.new_matcher_io())); } @@ -102,21 +88,15 @@ mod tests { create_file_link(); let link_f = get_dir_entry_for("test_data/links", "link-f"); - let matcher = NameMatcher::new("link?f", false).unwrap(); + let matcher = NameMatcher::new("link?f", false); let deps = FakeDependencies::new(); assert!(matcher.matches(&link_f, &mut deps.new_matcher_io())); } - #[test] - fn cant_create_with_invalid_pattern() { - let result = NameMatcher::new("a**c", false); - assert!(result.is_err()); - } - #[test] fn caseless_matching_with_wrong_case_returns_true() { let abbbc = get_dir_entry_for("test_data/simple", "abbbc"); - let matcher = NameMatcher::new("A*C", true).unwrap(); + let matcher = NameMatcher::new("A*C", true); let deps = FakeDependencies::new(); assert!(matcher.matches(&abbbc, &mut deps.new_matcher_io())); } @@ -124,7 +104,7 @@ mod tests { #[test] fn caseless_matching_with_right_case_returns_true() { let abbbc = get_dir_entry_for("test_data/simple", "abbbc"); - let matcher = NameMatcher::new("abb?c", true).unwrap(); + let matcher = NameMatcher::new("abb?c", true); let deps = FakeDependencies::new(); assert!(matcher.matches(&abbbc, &mut deps.new_matcher_io())); } @@ -132,7 +112,7 @@ mod tests { #[test] fn caseless_not_matching_returns_false() { let abbbc = get_dir_entry_for("test_data/simple", "abbbc"); - let matcher = NameMatcher::new("shouldn't match", true).unwrap(); + let matcher = NameMatcher::new("shouldn't match", true); let deps = FakeDependencies::new(); assert!(!matcher.matches(&abbbc, &mut deps.new_matcher_io())); } @@ -142,14 +122,8 @@ mod tests { create_file_link(); let link_f = get_dir_entry_for("test_data/links", "link-f"); - let matcher = NameMatcher::new("linK?f", true).unwrap(); + let matcher = NameMatcher::new("linK?f", true); let deps = FakeDependencies::new(); assert!(matcher.matches(&link_f, &mut deps.new_matcher_io())); } - - #[test] - fn caseless_cant_create_with_invalid_pattern() { - let result = NameMatcher::new("a**c", true); - assert!(result.is_err()); - } } diff --git a/src/lib.rs b/src/lib.rs index 7a60d7d..9f959d0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,12 +4,5 @@ // license that can be found in the LICENSE file or at // https://opensource.org/licenses/MIT. -extern crate glob; -extern crate regex; -extern crate walkdir; - -#[cfg(test)] -extern crate tempfile; - pub mod find; pub mod xargs;