parser, lexer: typed regexes support

lexer/src/lib.rs

@@ -143,6 +143,8 @@ pub enum Token<'a> {
     Slash,
     /// Not generated by Logos directly.
     Regex(Slice<'a>),
+    #[token("@/", |lex| { accept_operator(lex); parse_content::<true, '/'>(lex) })]
+    TypedRegex(Slice<'a>),
     #[token("%", accept_expression)]
     Percent,
     #[token("^", accept_expression)]

parser/src/ast.rs

@@ -38,6 +38,7 @@ pub enum Atom<'a> {
     BigInt(),
     BigFloat(),
     Regex(Slice<'a>),
+    TypedRegex(Slice<'a>),
 }
 
 #[derive(Debug)]
@@ -119,8 +120,8 @@ pub enum BinaryOperator {
     GtE,
     And,
     Or,
-    MatchRegex,
-    NotMatchRegex,
+    Matches,
+    MatchesNot,
     Add,
     Subtract,
     Multiply,
@@ -338,8 +339,8 @@ impl<'a> BinaryOperator {
             Token::LesserThan => Ok(Self::Lt),
             Token::GreaterOrEqualThan => Ok(Self::GtE),
             Token::LesserOrEqualThan => Ok(Self::LtE),
-            Token::Matching => Ok(Self::MatchRegex),
-            Token::NotMatching => Ok(Self::NotMatchRegex),
+            Token::Matching => Ok(Self::Matches),
+            Token::NotMatching => Ok(Self::MatchesNot),
             Token::Plus => Ok(Self::Add),
             Token::Minus => Ok(Self::Subtract),
             Token::Star => Ok(Self::Multiply),
@@ -486,7 +487,7 @@ impl BindingPower for BinaryOperator {
             }
             Self::And => binding_powers::BP_AND,
             Self::Or => binding_powers::BP_OR,
-            Self::MatchRegex | Self::NotMatchRegex => binding_powers::BP_MATCH,
+            Self::Matches | Self::MatchesNot => binding_powers::BP_MATCH,
             Self::Add => binding_powers::BP_ADDITION,
             Self::Subtract => binding_powers::BP_ADDITION,
             Self::Multiply => binding_powers::BP_MULTI,

parser/src/diagnostics.rs

@@ -49,8 +49,8 @@ pub enum ParsingError {
     UnclosedArrayAccess(Span),
     #[error("Expected operand to be a variable.")]
     OperatorExpectsVariable(Span),
-    #[error("Malformed expression.")]
-    InvalidExpression(Span),
+    #[error("Malformed expression: {}", .1)]
+    InvalidExpression(Span, String),
     #[error("Missing alternate branch in ternary expression.")]
     MissingTernaryOr(Span),
     #[error("Missing closing parenthesis in function call to `{}`.", .1)]
@@ -67,6 +67,8 @@ pub enum ParsingError {
     ExpectedBinaryOperator(Span),
     #[error("Expected a placing operation.")]
     ExpectedPlaceOperator(Span),
+    #[error("Typed regular expressions not accepted in this position.")]
+    UnexpectedTypedRegex(Span),
 }
 
 impl ParsingError {
@@ -100,7 +102,7 @@ impl ParsingError {
             Self::UnclosedParenthesisExpression(span) => Some(span.clone()),
             Self::UnclosedArrayAccess(span) => Some(span.clone()),
             Self::OperatorExpectsVariable(span) => Some(span.clone()),
-            Self::InvalidExpression(span) => Some(span.clone()),
+            Self::InvalidExpression(span, _) => Some(span.clone()),
             Self::MissingTernaryOr(span) => Some(span.clone()),
             Self::FunctionCallMissingParenthesis(span, _) => Some(span.clone()),
             Self::FunctionCallSeparatedIdent(span) => Some(span.clone()),
@@ -109,6 +111,7 @@ impl ParsingError {
             Self::ExpectedUnaryOperator(span) => Some(span.clone()),
             Self::ExpectedBinaryOperator(span) => Some(span.clone()),
             Self::ExpectedPlaceOperator(span) => Some(span.clone()),
+            Self::UnexpectedTypedRegex(span) => Some(span.clone()),
         }
     }
     fn hint(&self) -> Option<&'static str> {
@@ -139,6 +142,9 @@ impl ParsingError {
             Self::LexingError(LexingError::UnavailableOnGnu(_, _)) => {
                 Some("This item is not available in GNU-strict mode.")
             }
+            Self::UnexpectedTypedRegex(_) => Some(
+                "This is only valid in some contexts, like a right-hand assignment or a function argument.",
+            ),
             _ => None,
         }
     }

parser/src/idempotency.rs

@@ -372,8 +372,8 @@ impl Display for BinaryOperator {
             Self::GtE => write!(f, " >= "),
             Self::And => write!(f, " && "),
             Self::Or => write!(f, " || "),
-            Self::MatchRegex => write!(f, " ~ "),
-            Self::NotMatchRegex => write!(f, " !~ "),
+            Self::Matches => write!(f, " ~ "),
+            Self::MatchesNot => write!(f, " !~ "),
             Self::Add => write!(f, " + "),
             Self::Subtract => write!(f, " - "),
             Self::Multiply => write!(f, " * "),

parser/src/lex.rs

@@ -71,14 +71,16 @@ impl<'a> Lexer<'a> {
     }
 
     pub fn expect_identifier(&mut self) -> super::Result<lexer::Identifier<'a>> {
-        let Token::Identifier(name) = self.expect_with(
-            |t| matches!(t, Token::Identifier(_)),
-            "expected an identifier.".into(),
-        )?
-        else {
-            unreachable!()
-        };
-        Ok(name)
+        if let Some(Token::Identifier(ident)) =
+            self.next_if(|t| matches!(t, Token::Identifier(_)))?
+        {
+            Ok(ident)
+        } else {
+            Err(ParsingError::UnexpectedToken(
+                self.peeked_span().unwrap_or(self.span()),
+                "expected an identifier.".into(),
+            ))
+        }
     }
 
     pub fn consume(&mut self, token: &Token) -> bool {
@@ -103,9 +105,12 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    pub fn next_if(&mut self, f: impl FnOnce(&LexItem<'a>) -> bool) -> Option<LexItem<'a>> {
-        let next = self.inner.next_if(|(tok, _)| f(tok));
-        self.advance_span(next)
+    pub fn next_if(
+        &mut self,
+        f: impl FnOnce(&Token<'a>) -> bool,
+    ) -> lexer::Result<Option<Token<'a>>> {
+        let next = self.inner.next_if(|(tok, _)| tok.as_ref().is_ok_and(f));
+        self.advance_span(next).transpose()
     }
 
     pub fn expect_next(&mut self) -> super::Result<Token<'a>> {

parser/src/lib.rs

@@ -178,9 +178,9 @@ impl<'a> Parser<'a> {
             Token::BeginFilePattern => Ok(Right(SpecialPattern::BeginFile)),
             Token::EndFilePattern => Ok(Right(SpecialPattern::EndFile)),
             _ => {
-                let expr = self.parse_expression(lex)?;
+                let expr = self.parse_expression(lex, false)?;
                 Ok(Left(if lex.consume(&Token::Comma) {
-                    let expr_end = self.parse_expression(lex)?;
+                    let expr_end = self.parse_expression(lex, false)?;
                     RulePattern::Range(expr, expr_end)
                 } else {
                     RulePattern::Expression(expr)
@@ -226,7 +226,10 @@ impl<'a> Parser<'a> {
     ) -> Option<Result<SimpleStatement<'a>>> {
         let peek = lex.expect_peek().ok()?;
         if peek.is_expr_start() {
-            Some(self.parse_expression(lex).map(SimpleStatement::Expression))
+            Some(
+                self.parse_expression(lex, false)
+                    .map(SimpleStatement::Expression),
+            )
         } else {
             match peek {
                 token if let Some(name) = token.maps_to_command() => {
@@ -357,14 +360,14 @@ impl<'a> Parser<'a> {
                 Token::Continue => Statement::Continue,
                 Token::Return => Statement::Return(
                     (!lex.peek_with(Token::is_stmnt_or_block_end))
-                        .then(|| self.parse_expression(lex))
+                        .then(|| self.parse_expression(lex, true))
                         .transpose()?,
                 ),
                 Token::Next => Statement::Next,
                 Token::NextFile => Statement::NextFile,
                 Token::Exit => Statement::Exit(
                     (!lex.peek_with(Token::is_stmnt_or_block_end))
-                        .then(|| self.parse_expression(lex))
+                        .then(|| self.parse_expression(lex, false))
                         .transpose()?,
                 ),
                 _ => {
@@ -386,7 +389,7 @@ impl<'a> Parser<'a> {
             &Token::OpenParent,
             ParsingError::MissingParenthesisInStatement,
         )?;
-        let expr = self.parse_expression(lex)?;
+        let expr = self.parse_expression(lex, false)?;
         lex.expect(
             &Token::ClosedParent,
             ParsingError::UnclosedParenthesisInStatement,
@@ -402,7 +405,7 @@ impl<'a> Parser<'a> {
     ) -> Result<Statement<'a>> {
         lex.consume(&Token::Newline);
         let condition = (!lex.peek_is(&Token::Semicolon))
-            .then(|| self.parse_expression(lex))
+            .then(|| self.parse_expression(lex, false))
             .transpose()?;
         lex.expect(&Token::Semicolon, ParsingError::InvalidForLoop)?;
 
@@ -469,7 +472,7 @@ impl<'a> Parser<'a> {
     fn parse_case(&mut self, lex: &mut Lexer<'a>) -> Result<Atom<'a>> {
         lex.expect(&Token::Case, ParsingError::MissingSwitchBranch)?;
         let next = lex.expect_next()?;
-        let value = self.parse_atom(lex, next)?;
+        let value = self.parse_atom(lex, next, true)?;
         lex.expect(&Token::Colon, ParsingError::ColonMustFollowCase)?;
         match value {
             Atom::Variable(_) => Err(ParsingError::InvalidCaseValue(lex.span())),
@@ -506,16 +509,16 @@ impl<'a> Parser<'a> {
             return Ok(arguments);
         }
 
-        arguments.push(self.parse_expression(lex)?);
+        arguments.push(self.parse_expression(lex, true)?);
         while lex.consume(&Token::Comma) {
-            arguments.push(self.parse_expression(lex)?);
+            arguments.push(self.parse_expression(lex, true)?);
         }
         Ok(arguments)
     }
 
     fn parse_command_args(&mut self, lex: &mut Lexer<'a>) -> Result<Vec<'a, Expr<'a>>> {
         let mut arguments = Vec::new_in(self.arena);
-        let mut pratt = Pratt::new(self);
+        let mut pratt = Pratt::new(self, false);
         if !lex.peek_with(Token::is_expr_start) {
             return Ok(arguments);
         }
@@ -537,7 +540,7 @@ impl<'a> Parser<'a> {
             lex.next();
             Ok(Some((
                 redirection,
-                Pratt::new(self).parse_redirection(lex)?,
+                Pratt::new(self, false).parse_redirection(lex)?,
             )))
         } else {
             Ok(None)
@@ -550,7 +553,7 @@ impl<'a> Parser<'a> {
             return Err(ParsingError::OperatorExpectsVariable(lex.span()));
         };
         let index = if lex.consume(&Token::OpenBracket) {
-            let mut pratt = Pratt::new(self);
+            let mut pratt = Pratt::new(self, false);
             let first = pratt.parse(lex)?;
             Some(pratt.parse_array_index(lex, first)?)
         } else {
@@ -608,8 +611,8 @@ impl<'a> Parser<'a> {
     }
 
     #[tracing::instrument]
-    fn parse_expression(&mut self, lex: &mut Lexer<'a>) -> Result<Expr<'a>> {
-        Pratt::new(self).parse(lex)
+    fn parse_expression(&mut self, lex: &mut Lexer<'a>, typed_regex: bool) -> Result<Expr<'a>> {
+        Pratt::new(self, typed_regex).parse(lex)
     }
 
     #[tracing::instrument]
@@ -642,11 +645,18 @@ impl<'a> Parser<'a> {
     }
 
     #[tracing::instrument]
-    fn parse_atom(&self, lex: &mut Lexer<'a>, token: Token<'a>) -> Result<Atom<'a>> {
+    fn parse_atom(
+        &self,
+        lex: &mut Lexer<'a>,
+        token: Token<'a>,
+        typed_regex: bool,
+    ) -> Result<Atom<'a>> {
         match token {
             Token::Number(n) => Ok(Atom::Number(n)),
             Token::String(s) => Ok(Atom::String(s)),
             Token::Regex(r) => Ok(Atom::Regex(r)),
+            Token::TypedRegex(r) if typed_regex => Ok(Atom::TypedRegex(r)),
+            Token::TypedRegex(_) => Err(ParsingError::UnexpectedTypedRegex(lex.span())),
             token => match self.get_place(lex, token) {
                 Some(var) => Ok(Atom::Variable(var)),
                 None => Err(ParsingError::UnexpectedToken(

parser/src/pratt.rs

@@ -8,7 +8,7 @@ use lexer::Token;
 use crate::{
     IdentifierExt, Lexer, Parser, Result,
     ast::{
-        BinaryOperator, BinaryPlaceOperator, BindingPower, Expr, ExprNode, Getline, Place,
+        Atom, BinaryOperator, BinaryPlaceOperator, BindingPower, Expr, ExprNode, Getline, Place,
         Redirection, Ternary, UnaryOperator, UnaryPlaceOperator, Variable, WriteKind,
     },
     diagnostics::ParsingError,
@@ -17,11 +17,15 @@ use crate::{
 
 pub struct Pratt<'a, 'b> {
     parser: &'b mut Parser<'a>,
+    typed_regex: bool,
 }
 
 impl<'a, 'b> Pratt<'a, 'b> {
-    pub fn new(parser: &'b mut Parser<'a>) -> Self {
-        Self { parser }
+    pub fn new(parser: &'b mut Parser<'a>, typed_regex: bool) -> Self {
+        Self {
+            parser,
+            typed_regex,
+        }
     }
 
     pub fn parse(&mut self, lex: &mut Lexer<'a>) -> Result<Expr<'a>> {
@@ -62,6 +66,8 @@ impl<'a, 'b> Pratt<'a, 'b> {
             if delimiter(next) {
                 break;
             }
+            // Reset typed regex acceptance.
+            self.typed_regex = false;
             lhs = if let Ok(op) = UnaryPlaceOperator::parse_suffix(next, &span) {
                 if op.binding_power() < min_bp {
                     break;
@@ -100,16 +106,20 @@ impl<'a, 'b> Pratt<'a, 'b> {
     }
 
     fn parse_parenthesized(&mut self, lex: &mut Lexer<'a>) -> Result<Expr<'a>> {
-        let inner = self.parse(lex);
+        // I would consider this a gawk bug, but it's most likely a wontfix.
+        self.typed_regex = false;
+        let inner = self.parse(lex)?;
         lex.expect(
             &Token::ClosedParent,
             ParsingError::UnclosedParenthesisExpression,
-        )
-        .and(inner)
+        )?;
+        Ok(inner)
     }
 
     fn parse_prefix(&mut self, lex: &mut Lexer<'a>) -> Result<Expr<'a>> {
         let next = lex.expect_next()?;
+        // No prefix operator accepts them.
+        self.typed_regex = false;
         if let Ok(op) = UnaryPlaceOperator::parse_prefix(&next, &lex.span()) {
             let rhs = self.parse_expression(lex, op.binding_power())?;
             Ok(Expr::node(
@@ -120,13 +130,17 @@ impl<'a, 'b> Pratt<'a, 'b> {
             let rhs = self.parse_expression(lex, op.binding_power())?;
             Ok(Expr::node(op.expr(rhs), self.parser.arena))
         } else {
-            Err(ParsingError::InvalidExpression(lex.span()))
+            Err(ParsingError::InvalidExpression(
+                lex.span(),
+                "expected a valid prefix operator".into(),
+            ))
         }
     }
 
     fn parse_prefix_getline(&mut self, lex: &mut Lexer<'a>) -> Result<Expr<'a>> {
         // Consumes with maximum precedence the following place and/or
-        // redirection reading from file.
+        // redirection reading from file. Does not accept typed regexes.
+        self.typed_regex = false;
         let place = if lex.peek_with(Token::is_place) {
             Some(Place::promote_from(
                 self.parse_redirection(lex)?,
@@ -163,9 +177,12 @@ impl<'a, 'b> Pratt<'a, 'b> {
             self.parser
                 .parse_function_call(lex, name.qualify(self.parser.namespace), lex.span())
         } else {
-            match self.parser.parse_atom(lex, next) {
+            match self.parser.parse_atom(lex, next, self.typed_regex) {
                 Ok(atom) => Ok(Expr::leaf(atom)),
-                Err(_) => Err(ParsingError::InvalidExpression(lex.span())),
+                Err(ParsingError::UnexpectedToken(_, str)) => {
+                    Err(ParsingError::InvalidExpression(lex.span(), str))
+                }
+                Err(e) => Err(e),
             }
         }
     }
@@ -176,7 +193,9 @@ impl<'a, 'b> Pratt<'a, 'b> {
         op: BinaryOperator,
         lhs: Expr<'a>,
     ) -> Result<Expr<'a>> {
+        self.typecheck(lex, &lhs)?;
         lex.consume_with(|_| op != BinaryOperator::Concat);
+        self.typed_regex = matches!(op, BinaryOperator::Matches | BinaryOperator::MatchesNot);
 
         let rhs = self.parse_expression(lex, op.binding_power().1)?;
         Ok(Expr::node(op.expr(lhs, rhs), self.parser.arena))
@@ -189,7 +208,25 @@ impl<'a, 'b> Pratt<'a, 'b> {
         place: Place<'a>,
     ) -> Result<Expr<'a>> {
         lex.next();
-        let mut rhs = self.parse_expression(lex, op.binding_power().1)?;
+        self.typed_regex = matches!(op, BinaryPlaceOperator::Assignment);
+        // Assignment expressions can consume with maximum precedence a
+        // following typed regex, so it bypasses ternaries (the only operations
+        // with lesser binding power); i.e., we parse `x = @/a/ ? a : b` into
+        // `(?: (= x @/a/) a b)`. This is generally true for all positions of
+        // typed regexes, but only an edge case here.
+        let mut rhs = if self.typed_regex
+            && let Some(Token::TypedRegex(slice)) =
+                lex.next_if(|t| matches!(t, Token::TypedRegex(_)))?
+        {
+            let lhs = Expr::Leaf(Atom::TypedRegex(slice));
+            // We fold it in order to catch invalid cases, like `x = @/a/ + 1`.
+            // Also allows us to bypass ternaries without binding power hacks.
+            self.fold_rhs(lex, lhs, op.binding_power().0, |t| {
+                t == &Token::QuestionMark
+            })?
+        } else {
+            self.parse_expression(lex, op.binding_power().1)?
+        };
         if op == BinaryPlaceOperator::ArrayAccess {
             if !matches!(place, Place::Variable(_)) {
                 return Err(ParsingError::OperatorExpectsVariable(lex.span()));
@@ -218,6 +255,9 @@ impl<'a, 'b> Pratt<'a, 'b> {
     }
 
     fn parse_ternary(&mut self, lex: &mut Lexer<'a>, lhs: Expr<'a>) -> Result<Expr<'a>> {
+        // There should be no need to typecheck lhs since there is no way it
+        // wasn't caught first, but checking is cheap, so we make sure.
+        self.typecheck(lex, &lhs)?;
         let right_bp = Ternary.binding_power().1;
         lex.next();
         let then_branch = self.parse_expression(lex, right_bp)?;
@@ -261,4 +301,12 @@ impl<'a, 'b> Pratt<'a, 'b> {
     pub fn parse_redirection(&mut self, lex: &mut Lexer<'a>) -> Result<Expr<'a>> {
         self.parse_expression(lex, BinaryOperator::Concat.binding_power().1 - 1)
     }
+
+    fn typecheck(&self, lex: &mut Lexer<'a>, expr: &Expr<'a>) -> Result<()> {
+        if matches!(expr, Expr::Leaf(Atom::TypedRegex(_))) {
+            Err(ParsingError::UnexpectedTypedRegex(lex.span()))
+        } else {
+            Ok(())
+        }
+    }
 }

parser/src/sexpr.rs

@@ -256,6 +256,7 @@ impl Debug for Atom<'_> {
             Self::String(str) => write!(f, "{str:?}"),
             Self::Number(num) => write!(f, "{num}"),
             Self::Regex(rgx) => write!(f, "/{rgx}/"),
+            Self::TypedRegex(rgx) => write!(f, "@/{rgx}/"),
             Self::BigInt() | Self::BigFloat() => unimplemented!(),
         }
     }