diff --git a/lexer/src/lib.rs b/lexer/src/lib.rs index e98a20b..7b2fb5d 100644 --- a/lexer/src/lib.rs +++ b/lexer/src/lib.rs @@ -143,6 +143,8 @@ pub enum Token<'a> { Slash, /// Not generated by Logos directly. Regex(Slice<'a>), + #[token("@/", |lex| { accept_operator(lex); parse_content::(lex) })] + TypedRegex(Slice<'a>), #[token("%", accept_expression)] Percent, #[token("^", accept_expression)] diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 0d43528..f94b3cf 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -38,6 +38,7 @@ pub enum Atom<'a> { BigInt(), BigFloat(), Regex(Slice<'a>), + TypedRegex(Slice<'a>), } #[derive(Debug)] @@ -119,8 +120,8 @@ pub enum BinaryOperator { GtE, And, Or, - MatchRegex, - NotMatchRegex, + Matches, + MatchesNot, Add, Subtract, Multiply, @@ -338,8 +339,8 @@ impl<'a> BinaryOperator { Token::LesserThan => Ok(Self::Lt), Token::GreaterOrEqualThan => Ok(Self::GtE), Token::LesserOrEqualThan => Ok(Self::LtE), - Token::Matching => Ok(Self::MatchRegex), - Token::NotMatching => Ok(Self::NotMatchRegex), + Token::Matching => Ok(Self::Matches), + Token::NotMatching => Ok(Self::MatchesNot), Token::Plus => Ok(Self::Add), Token::Minus => Ok(Self::Subtract), Token::Star => Ok(Self::Multiply), @@ -486,7 +487,7 @@ impl BindingPower for BinaryOperator { } Self::And => binding_powers::BP_AND, Self::Or => binding_powers::BP_OR, - Self::MatchRegex | Self::NotMatchRegex => binding_powers::BP_MATCH, + Self::Matches | Self::MatchesNot => binding_powers::BP_MATCH, Self::Add => binding_powers::BP_ADDITION, Self::Subtract => binding_powers::BP_ADDITION, Self::Multiply => binding_powers::BP_MULTI, diff --git a/parser/src/diagnostics.rs b/parser/src/diagnostics.rs index da53d6c..773bdb4 100644 --- a/parser/src/diagnostics.rs +++ b/parser/src/diagnostics.rs @@ -49,8 +49,8 @@ pub enum ParsingError { UnclosedArrayAccess(Span), #[error("Expected operand to be a variable.")] OperatorExpectsVariable(Span), - #[error("Malformed expression.")] - InvalidExpression(Span), + #[error("Malformed expression: {}", .1)] + InvalidExpression(Span, String), #[error("Missing alternate branch in ternary expression.")] MissingTernaryOr(Span), #[error("Missing closing parenthesis in function call to `{}`.", .1)] @@ -67,6 +67,8 @@ pub enum ParsingError { ExpectedBinaryOperator(Span), #[error("Expected a placing operation.")] ExpectedPlaceOperator(Span), + #[error("Typed regular expressions not accepted in this position.")] + UnexpectedTypedRegex(Span), } impl ParsingError { @@ -100,7 +102,7 @@ impl ParsingError { Self::UnclosedParenthesisExpression(span) => Some(span.clone()), Self::UnclosedArrayAccess(span) => Some(span.clone()), Self::OperatorExpectsVariable(span) => Some(span.clone()), - Self::InvalidExpression(span) => Some(span.clone()), + Self::InvalidExpression(span, _) => Some(span.clone()), Self::MissingTernaryOr(span) => Some(span.clone()), Self::FunctionCallMissingParenthesis(span, _) => Some(span.clone()), Self::FunctionCallSeparatedIdent(span) => Some(span.clone()), @@ -109,6 +111,7 @@ impl ParsingError { Self::ExpectedUnaryOperator(span) => Some(span.clone()), Self::ExpectedBinaryOperator(span) => Some(span.clone()), Self::ExpectedPlaceOperator(span) => Some(span.clone()), + Self::UnexpectedTypedRegex(span) => Some(span.clone()), } } fn hint(&self) -> Option<&'static str> { @@ -139,6 +142,9 @@ impl ParsingError { Self::LexingError(LexingError::UnavailableOnGnu(_, _)) => { Some("This item is not available in GNU-strict mode.") } + Self::UnexpectedTypedRegex(_) => Some( + "This is only valid in some contexts, like a right-hand assignment or a function argument.", + ), _ => None, } } diff --git a/parser/src/idempotency.rs b/parser/src/idempotency.rs index 81b9ff1..ddedf39 100644 --- a/parser/src/idempotency.rs +++ b/parser/src/idempotency.rs @@ -372,8 +372,8 @@ impl Display for BinaryOperator { Self::GtE => write!(f, " >= "), Self::And => write!(f, " && "), Self::Or => write!(f, " || "), - Self::MatchRegex => write!(f, " ~ "), - Self::NotMatchRegex => write!(f, " !~ "), + Self::Matches => write!(f, " ~ "), + Self::MatchesNot => write!(f, " !~ "), Self::Add => write!(f, " + "), Self::Subtract => write!(f, " - "), Self::Multiply => write!(f, " * "), diff --git a/parser/src/lex.rs b/parser/src/lex.rs index d7e9a22..c7a3bb6 100644 --- a/parser/src/lex.rs +++ b/parser/src/lex.rs @@ -71,14 +71,16 @@ impl<'a> Lexer<'a> { } pub fn expect_identifier(&mut self) -> super::Result> { - let Token::Identifier(name) = self.expect_with( - |t| matches!(t, Token::Identifier(_)), - "expected an identifier.".into(), - )? - else { - unreachable!() - }; - Ok(name) + if let Some(Token::Identifier(ident)) = + self.next_if(|t| matches!(t, Token::Identifier(_)))? + { + Ok(ident) + } else { + Err(ParsingError::UnexpectedToken( + self.peeked_span().unwrap_or(self.span()), + "expected an identifier.".into(), + )) + } } pub fn consume(&mut self, token: &Token) -> bool { @@ -103,9 +105,12 @@ impl<'a> Lexer<'a> { } } - pub fn next_if(&mut self, f: impl FnOnce(&LexItem<'a>) -> bool) -> Option> { - let next = self.inner.next_if(|(tok, _)| f(tok)); - self.advance_span(next) + pub fn next_if( + &mut self, + f: impl FnOnce(&Token<'a>) -> bool, + ) -> lexer::Result>> { + let next = self.inner.next_if(|(tok, _)| tok.as_ref().is_ok_and(f)); + self.advance_span(next).transpose() } pub fn expect_next(&mut self) -> super::Result> { diff --git a/parser/src/lib.rs b/parser/src/lib.rs index b75f119..2576ad1 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -178,9 +178,9 @@ impl<'a> Parser<'a> { Token::BeginFilePattern => Ok(Right(SpecialPattern::BeginFile)), Token::EndFilePattern => Ok(Right(SpecialPattern::EndFile)), _ => { - let expr = self.parse_expression(lex)?; + let expr = self.parse_expression(lex, false)?; Ok(Left(if lex.consume(&Token::Comma) { - let expr_end = self.parse_expression(lex)?; + let expr_end = self.parse_expression(lex, false)?; RulePattern::Range(expr, expr_end) } else { RulePattern::Expression(expr) @@ -226,7 +226,10 @@ impl<'a> Parser<'a> { ) -> Option>> { let peek = lex.expect_peek().ok()?; if peek.is_expr_start() { - Some(self.parse_expression(lex).map(SimpleStatement::Expression)) + Some( + self.parse_expression(lex, false) + .map(SimpleStatement::Expression), + ) } else { match peek { token if let Some(name) = token.maps_to_command() => { @@ -357,14 +360,14 @@ impl<'a> Parser<'a> { Token::Continue => Statement::Continue, Token::Return => Statement::Return( (!lex.peek_with(Token::is_stmnt_or_block_end)) - .then(|| self.parse_expression(lex)) + .then(|| self.parse_expression(lex, true)) .transpose()?, ), Token::Next => Statement::Next, Token::NextFile => Statement::NextFile, Token::Exit => Statement::Exit( (!lex.peek_with(Token::is_stmnt_or_block_end)) - .then(|| self.parse_expression(lex)) + .then(|| self.parse_expression(lex, false)) .transpose()?, ), _ => { @@ -386,7 +389,7 @@ impl<'a> Parser<'a> { &Token::OpenParent, ParsingError::MissingParenthesisInStatement, )?; - let expr = self.parse_expression(lex)?; + let expr = self.parse_expression(lex, false)?; lex.expect( &Token::ClosedParent, ParsingError::UnclosedParenthesisInStatement, @@ -402,7 +405,7 @@ impl<'a> Parser<'a> { ) -> Result> { lex.consume(&Token::Newline); let condition = (!lex.peek_is(&Token::Semicolon)) - .then(|| self.parse_expression(lex)) + .then(|| self.parse_expression(lex, false)) .transpose()?; lex.expect(&Token::Semicolon, ParsingError::InvalidForLoop)?; @@ -469,7 +472,7 @@ impl<'a> Parser<'a> { fn parse_case(&mut self, lex: &mut Lexer<'a>) -> Result> { lex.expect(&Token::Case, ParsingError::MissingSwitchBranch)?; let next = lex.expect_next()?; - let value = self.parse_atom(lex, next)?; + let value = self.parse_atom(lex, next, true)?; lex.expect(&Token::Colon, ParsingError::ColonMustFollowCase)?; match value { Atom::Variable(_) => Err(ParsingError::InvalidCaseValue(lex.span())), @@ -506,16 +509,16 @@ impl<'a> Parser<'a> { return Ok(arguments); } - arguments.push(self.parse_expression(lex)?); + arguments.push(self.parse_expression(lex, true)?); while lex.consume(&Token::Comma) { - arguments.push(self.parse_expression(lex)?); + arguments.push(self.parse_expression(lex, true)?); } Ok(arguments) } fn parse_command_args(&mut self, lex: &mut Lexer<'a>) -> Result>> { let mut arguments = Vec::new_in(self.arena); - let mut pratt = Pratt::new(self); + let mut pratt = Pratt::new(self, false); if !lex.peek_with(Token::is_expr_start) { return Ok(arguments); } @@ -537,7 +540,7 @@ impl<'a> Parser<'a> { lex.next(); Ok(Some(( redirection, - Pratt::new(self).parse_redirection(lex)?, + Pratt::new(self, false).parse_redirection(lex)?, ))) } else { Ok(None) @@ -550,7 +553,7 @@ impl<'a> Parser<'a> { return Err(ParsingError::OperatorExpectsVariable(lex.span())); }; let index = if lex.consume(&Token::OpenBracket) { - let mut pratt = Pratt::new(self); + let mut pratt = Pratt::new(self, false); let first = pratt.parse(lex)?; Some(pratt.parse_array_index(lex, first)?) } else { @@ -608,8 +611,8 @@ impl<'a> Parser<'a> { } #[tracing::instrument] - fn parse_expression(&mut self, lex: &mut Lexer<'a>) -> Result> { - Pratt::new(self).parse(lex) + fn parse_expression(&mut self, lex: &mut Lexer<'a>, typed_regex: bool) -> Result> { + Pratt::new(self, typed_regex).parse(lex) } #[tracing::instrument] @@ -642,11 +645,18 @@ impl<'a> Parser<'a> { } #[tracing::instrument] - fn parse_atom(&self, lex: &mut Lexer<'a>, token: Token<'a>) -> Result> { + fn parse_atom( + &self, + lex: &mut Lexer<'a>, + token: Token<'a>, + typed_regex: bool, + ) -> Result> { match token { Token::Number(n) => Ok(Atom::Number(n)), Token::String(s) => Ok(Atom::String(s)), Token::Regex(r) => Ok(Atom::Regex(r)), + Token::TypedRegex(r) if typed_regex => Ok(Atom::TypedRegex(r)), + Token::TypedRegex(_) => Err(ParsingError::UnexpectedTypedRegex(lex.span())), token => match self.get_place(lex, token) { Some(var) => Ok(Atom::Variable(var)), None => Err(ParsingError::UnexpectedToken( diff --git a/parser/src/pratt.rs b/parser/src/pratt.rs index a41879f..10c3f31 100644 --- a/parser/src/pratt.rs +++ b/parser/src/pratt.rs @@ -8,7 +8,7 @@ use lexer::Token; use crate::{ IdentifierExt, Lexer, Parser, Result, ast::{ - BinaryOperator, BinaryPlaceOperator, BindingPower, Expr, ExprNode, Getline, Place, + Atom, BinaryOperator, BinaryPlaceOperator, BindingPower, Expr, ExprNode, Getline, Place, Redirection, Ternary, UnaryOperator, UnaryPlaceOperator, Variable, WriteKind, }, diagnostics::ParsingError, @@ -17,11 +17,15 @@ use crate::{ pub struct Pratt<'a, 'b> { parser: &'b mut Parser<'a>, + typed_regex: bool, } impl<'a, 'b> Pratt<'a, 'b> { - pub fn new(parser: &'b mut Parser<'a>) -> Self { - Self { parser } + pub fn new(parser: &'b mut Parser<'a>, typed_regex: bool) -> Self { + Self { + parser, + typed_regex, + } } pub fn parse(&mut self, lex: &mut Lexer<'a>) -> Result> { @@ -62,6 +66,8 @@ impl<'a, 'b> Pratt<'a, 'b> { if delimiter(next) { break; } + // Reset typed regex acceptance. + self.typed_regex = false; lhs = if let Ok(op) = UnaryPlaceOperator::parse_suffix(next, &span) { if op.binding_power() < min_bp { break; @@ -100,16 +106,20 @@ impl<'a, 'b> Pratt<'a, 'b> { } fn parse_parenthesized(&mut self, lex: &mut Lexer<'a>) -> Result> { - let inner = self.parse(lex); + // I would consider this a gawk bug, but it's most likely a wontfix. + self.typed_regex = false; + let inner = self.parse(lex)?; lex.expect( &Token::ClosedParent, ParsingError::UnclosedParenthesisExpression, - ) - .and(inner) + )?; + Ok(inner) } fn parse_prefix(&mut self, lex: &mut Lexer<'a>) -> Result> { let next = lex.expect_next()?; + // No prefix operator accepts them. + self.typed_regex = false; if let Ok(op) = UnaryPlaceOperator::parse_prefix(&next, &lex.span()) { let rhs = self.parse_expression(lex, op.binding_power())?; Ok(Expr::node( @@ -120,13 +130,17 @@ impl<'a, 'b> Pratt<'a, 'b> { let rhs = self.parse_expression(lex, op.binding_power())?; Ok(Expr::node(op.expr(rhs), self.parser.arena)) } else { - Err(ParsingError::InvalidExpression(lex.span())) + Err(ParsingError::InvalidExpression( + lex.span(), + "expected a valid prefix operator".into(), + )) } } fn parse_prefix_getline(&mut self, lex: &mut Lexer<'a>) -> Result> { // Consumes with maximum precedence the following place and/or - // redirection reading from file. + // redirection reading from file. Does not accept typed regexes. + self.typed_regex = false; let place = if lex.peek_with(Token::is_place) { Some(Place::promote_from( self.parse_redirection(lex)?, @@ -163,9 +177,12 @@ impl<'a, 'b> Pratt<'a, 'b> { self.parser .parse_function_call(lex, name.qualify(self.parser.namespace), lex.span()) } else { - match self.parser.parse_atom(lex, next) { + match self.parser.parse_atom(lex, next, self.typed_regex) { Ok(atom) => Ok(Expr::leaf(atom)), - Err(_) => Err(ParsingError::InvalidExpression(lex.span())), + Err(ParsingError::UnexpectedToken(_, str)) => { + Err(ParsingError::InvalidExpression(lex.span(), str)) + } + Err(e) => Err(e), } } } @@ -176,7 +193,9 @@ impl<'a, 'b> Pratt<'a, 'b> { op: BinaryOperator, lhs: Expr<'a>, ) -> Result> { + self.typecheck(lex, &lhs)?; lex.consume_with(|_| op != BinaryOperator::Concat); + self.typed_regex = matches!(op, BinaryOperator::Matches | BinaryOperator::MatchesNot); let rhs = self.parse_expression(lex, op.binding_power().1)?; Ok(Expr::node(op.expr(lhs, rhs), self.parser.arena)) @@ -189,7 +208,25 @@ impl<'a, 'b> Pratt<'a, 'b> { place: Place<'a>, ) -> Result> { lex.next(); - let mut rhs = self.parse_expression(lex, op.binding_power().1)?; + self.typed_regex = matches!(op, BinaryPlaceOperator::Assignment); + // Assignment expressions can consume with maximum precedence a + // following typed regex, so it bypasses ternaries (the only operations + // with lesser binding power); i.e., we parse `x = @/a/ ? a : b` into + // `(?: (= x @/a/) a b)`. This is generally true for all positions of + // typed regexes, but only an edge case here. + let mut rhs = if self.typed_regex + && let Some(Token::TypedRegex(slice)) = + lex.next_if(|t| matches!(t, Token::TypedRegex(_)))? + { + let lhs = Expr::Leaf(Atom::TypedRegex(slice)); + // We fold it in order to catch invalid cases, like `x = @/a/ + 1`. + // Also allows us to bypass ternaries without binding power hacks. + self.fold_rhs(lex, lhs, op.binding_power().0, |t| { + t == &Token::QuestionMark + })? + } else { + self.parse_expression(lex, op.binding_power().1)? + }; if op == BinaryPlaceOperator::ArrayAccess { if !matches!(place, Place::Variable(_)) { return Err(ParsingError::OperatorExpectsVariable(lex.span())); @@ -218,6 +255,9 @@ impl<'a, 'b> Pratt<'a, 'b> { } fn parse_ternary(&mut self, lex: &mut Lexer<'a>, lhs: Expr<'a>) -> Result> { + // There should be no need to typecheck lhs since there is no way it + // wasn't caught first, but checking is cheap, so we make sure. + self.typecheck(lex, &lhs)?; let right_bp = Ternary.binding_power().1; lex.next(); let then_branch = self.parse_expression(lex, right_bp)?; @@ -261,4 +301,12 @@ impl<'a, 'b> Pratt<'a, 'b> { pub fn parse_redirection(&mut self, lex: &mut Lexer<'a>) -> Result> { self.parse_expression(lex, BinaryOperator::Concat.binding_power().1 - 1) } + + fn typecheck(&self, lex: &mut Lexer<'a>, expr: &Expr<'a>) -> Result<()> { + if matches!(expr, Expr::Leaf(Atom::TypedRegex(_))) { + Err(ParsingError::UnexpectedTypedRegex(lex.span())) + } else { + Ok(()) + } + } } diff --git a/parser/src/sexpr.rs b/parser/src/sexpr.rs index cc087d6..5588cc8 100644 --- a/parser/src/sexpr.rs +++ b/parser/src/sexpr.rs @@ -256,6 +256,7 @@ impl Debug for Atom<'_> { Self::String(str) => write!(f, "{str:?}"), Self::Number(num) => write!(f, "{num}"), Self::Regex(rgx) => write!(f, "/{rgx}/"), + Self::TypedRegex(rgx) => write!(f, "@/{rgx}/"), Self::BigInt() | Self::BigFloat() => unimplemented!(), } }