diff --git a/lexer/src/lib.rs b/lexer/src/lib.rs index 7b2fb5d..25496ba 100644 --- a/lexer/src/lib.rs +++ b/lexer/src/lib.rs @@ -47,14 +47,14 @@ pub enum Token<'a> { BeginFilePattern, #[token("ENDFILE", |lex| parse_non_posix_keyword(lex, Token::EndFilePattern))] EndFilePattern, - #[token("@load \"", parse_directive)] - LoadDirective(Slice<'a>), - #[token("@include \"", parse_directive)] - IncludeDirective(Slice<'a>), - #[token("@nsinclude \"", parse_non_posix_directive)] - NsIncludeDirective(Slice<'a>), - #[regex("@namespace \"(?&identifier)\"", parse_namespace_directive)] - NamespaceDirective(&'a str), + #[token("@load")] + LoadDirective, + #[token("@include")] + IncludeDirective, + #[token("@nsinclude", parse_non_posix_operator)] + NsIncludeDirective, + #[regex("@namespace", parse_non_posix_operator)] + NamespaceDirective, #[token("@concurrent", parse_non_gnu_operator)] ConcurrentDirective, #[token("if", accept_expression)] @@ -130,9 +130,12 @@ pub enum Token<'a> { RlengthVariable, #[token("ENVIRON", accept_expression)] EnvironVariable, - #[regex("(?&identifier)", Identifier::without_namespace)] - #[regex(r"(?&identifier)::(?&identifier)", Identifier::with_namespace)] + #[regex("(?&identifier)", Identifier::without_namespace::<0>)] + #[regex(r"(?&identifier)::(?&identifier)", Identifier::with_namespace::<0>)] Identifier(Identifier<'a>), + #[regex("@(?&identifier)", parse_indirect_call::)] + #[regex(r"@(?&identifier)::(?&identifier)", parse_indirect_call::)] + IndirectCall(Identifier<'a>), #[token("+", accept_expression)] Plus, #[token("-", accept_expression)] @@ -333,29 +336,6 @@ fn parse_regex_or_op<'a>(lex: &mut logos::Lexer<'a, Token<'a>>) -> Result(lex: &mut Lexer<'a>) -> Result> { - accept_expression(lex); - parse_content::(lex) -} - -fn parse_non_posix_directive<'a>(lex: &mut Lexer<'a>) -> Result> { - if lex.extras.posix_strict { - Err(LexingError::non_posix(lex)) - } else { - parse_directive(lex) - } -} - -fn parse_namespace_directive<'a>(lex: &mut Lexer<'a>) -> Result<&'a str> { - if lex.extras.posix_strict { - Err(LexingError::non_posix(lex)) - } else { - accept_expression(lex); - let offset = "@namespace \"".len(); - Ok(parse_ident(lex, offset..lex.slice().len() - 1)) - } -} - fn parse_content<'a, const REGEX: bool, const DELIMITER: char>( lex: &mut Lexer<'a>, ) -> Result> { @@ -456,7 +436,7 @@ fn parse_float(lex: &mut Lexer<'_>) -> f64 { fn parse_non_posix_keyword<'a>(lex: &mut Lexer<'a>, other: Token<'a>) -> Token<'a> { if lex.extras.posix_strict { - Token::Identifier(Identifier::without_namespace(lex)) + Token::Identifier(Identifier::without_namespace::<0>(lex)) } else { accept_expression(lex); other @@ -481,22 +461,32 @@ fn parse_non_gnu_operator(lex: &mut Lexer<'_>) -> Result<()> { } } +fn parse_indirect_call<'a, const QUALIFIED: bool>(lex: &mut Lexer<'a>) -> Result> { + if lex.extras.posix_strict { + Err(LexingError::non_posix(lex)) + } else if QUALIFIED { + Identifier::with_namespace::<1>(lex) + } else { + Ok(Identifier::without_namespace::<1>(lex)) + } +} + impl<'a> Identifier<'a> { - fn without_namespace(lex: &mut Lexer<'a>) -> Self { + fn without_namespace(lex: &mut Lexer<'a>) -> Self { Self { namespace: None, - literal: parse_ident(lex, ..), + literal: parse_ident(lex, SKIP..), } } - fn with_namespace(lex: &mut Lexer<'a>) -> Result { + fn with_namespace(lex: &mut Lexer<'a>) -> Result { if lex.extras.posix_strict { Err(LexingError::non_posix(lex)) } else { // SAFETY: The regex matching ensures it is present and well-formed. let separator = unsafe { memchr(b':', lex.slice()).unwrap_unchecked() }; Ok(Self { - namespace: Some(parse_ident(lex, ..separator)), + namespace: Some(parse_ident(lex, SKIP..separator)), literal: parse_ident(lex, separator + 2..), }) } diff --git a/lexer/src/tests.rs b/lexer/src/tests.rs index c67c2cc..a5d21d1 100644 --- a/lexer/src/tests.rs +++ b/lexer/src/tests.rs @@ -153,17 +153,42 @@ fn lexer_test_directive_escaping() { assert_eq!( &lex(str, &arena, false, false), &[ - Token::IncludeDirective(b"aa\"a\ta".into()), - Token::NsIncludeDirective(b"b\"\nb".into()) + Token::IncludeDirective, + Token::String(b"aa\"a\ta".into()), + Token::NsIncludeDirective, + Token::String(b"b\"\nb".into()) ] ); } #[test] -#[should_panic] fn lexer_test_ident_rules_non_posix() { let arena = Bump::new(); - lex(b"@namespace \"1a\"; a::1a", &arena, false, false); + assert_eq!( + &lex(b"1a::a a::1a _a", &arena, false, false), + &[ + Token::Number(1.), + Token::Identifier(Identifier { + namespace: Some("a"), + literal: "a" + }), + Token::Identifier(Identifier { + namespace: None, + literal: "a" + }), + Token::Colon, + Token::Colon, + Token::Number(1.), + Token::Identifier(Identifier { + namespace: None, + literal: "a" + }), + Token::Identifier(Identifier { + namespace: None, + literal: "_a" + }) + ] + ); } #[test] @@ -186,7 +211,8 @@ fn lexer_test_general_tokens() { &lex(str, &arena, false, false), &[ Token::Newline, - Token::LoadDirective(b"lib1.so.1".into()), + Token::LoadDirective, + Token::String(b"lib1.so.1".into()), Token::Newline, Token::BeginPattern, Token::OpenBrace, diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 4d54c5f..a361196 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -93,6 +93,7 @@ pub type Pattern<'a> = Either, SpecialPattern>; #[derive(Debug)] pub enum ExprNode<'a> { FunctionCall(Identifier<'a>, Vec<'a, Expr<'a>>), + IndirectCall(Variable<'a>, Vec<'a, Expr<'a>>), UnaryOperation(UnaryOperator, Expr<'a>), BinaryOperation(BinaryOperator, Expr<'a>, Expr<'a>), UnaryPlaceOperation(UnaryPlaceOperator, Place<'a>), @@ -556,6 +557,13 @@ impl Debug for Expr<'_> { } write!(f, ")") } + ExprNode::IndirectCall(ident, args) => { + write!(f, "(@{ident:?}")?; + for arg in args { + write!(f, " {arg:?}")?; + } + write!(f, ")") + } ExprNode::UnaryOperation(op, a) => write!(f, "({op:?} {a:?})"), ExprNode::BinaryOperation(op, a, b) => write!(f, "({op:?} {a:?} {b:?})"), ExprNode::BinaryPlaceOperation(op, a, b) => write!(f, "({op:?} {a:?} {b:?})"), diff --git a/parser/src/diagnostics.rs b/parser/src/diagnostics.rs index 773bdb4..595be30 100644 --- a/parser/src/diagnostics.rs +++ b/parser/src/diagnostics.rs @@ -53,8 +53,8 @@ pub enum ParsingError { InvalidExpression(Span, String), #[error("Missing alternate branch in ternary expression.")] MissingTernaryOr(Span), - #[error("Missing closing parenthesis in function call to `{}`.", .1)] - FunctionCallMissingParenthesis(Span, String), + #[error("Missing closing parenthesis in function call.")] + FunctionCallMissingParenthesis(Span), #[error("Functions calls must have their name yuxtaposed to the parenthesis `(`.")] FunctionCallSeparatedIdent(Span), #[error("Missing closing parenthesis `(` in function call to `{}`.", .1)] @@ -69,6 +69,10 @@ pub enum ParsingError { ExpectedPlaceOperator(Span), #[error("Typed regular expressions not accepted in this position.")] UnexpectedTypedRegex(Span), + #[error("Can't call non-function, special variable `{}`.", .1)] + SpecialVariableCall(Span, String), + #[error("Can't use special variable `{}` for indirect function call.", .1)] + SpecialVariableIndirectCall(Span, String), } impl ParsingError { @@ -104,7 +108,7 @@ impl ParsingError { Self::OperatorExpectsVariable(span) => Some(span.clone()), Self::InvalidExpression(span, _) => Some(span.clone()), Self::MissingTernaryOr(span) => Some(span.clone()), - Self::FunctionCallMissingParenthesis(span, _) => Some(span.clone()), + Self::FunctionCallMissingParenthesis(span) => Some(span.clone()), Self::FunctionCallSeparatedIdent(span) => Some(span.clone()), Self::FunctionCallUnclosed(span, _) => Some(span.clone()), Self::ExpectedIdentifier(span) => Some(span.clone()), @@ -112,6 +116,8 @@ impl ParsingError { Self::ExpectedBinaryOperator(span) => Some(span.clone()), Self::ExpectedPlaceOperator(span) => Some(span.clone()), Self::UnexpectedTypedRegex(span) => Some(span.clone()), + Self::SpecialVariableCall(span, _) => Some(span.clone()), + Self::SpecialVariableIndirectCall(span, _) => Some(span.clone()), } } fn hint(&self) -> Option<&'static str> { diff --git a/parser/src/idempotency.rs b/parser/src/idempotency.rs index ddedf39..4eaee7b 100644 --- a/parser/src/idempotency.rs +++ b/parser/src/idempotency.rs @@ -277,6 +277,11 @@ impl Display for ExprNode<'_> { write_args(f, args, indent)?; write!(f, ")") } + Self::IndirectCall(var, args) => { + write!(f, "@{var}(")?; + write_args(f, args, indent)?; + write!(f, ")") + } Self::UnaryOperation(op, x) => { let bp = op.binding_power(); let child_w = encode(indent, bp.saturating_add(1)); diff --git a/parser/src/lex.rs b/parser/src/lex.rs index c7a3bb6..daf3c50 100644 --- a/parser/src/lex.rs +++ b/parser/src/lex.rs @@ -6,13 +6,15 @@ use std::{fmt::Debug, iter::Peekable}; use bumpalo::Bump; -use lexer::{LexingError, Span, SpannedIter, Token}; +use lexer::{Identifier, LexingError, Slice, Span, SpannedIter, Token}; use crate::{ ParsingError, ast::{Command, SpecialPattern}, }; +use super::Result; + pub struct Lexer<'a> { inner: Peekable>>, span: Span, @@ -48,7 +50,7 @@ impl<'a> Lexer<'a> { &mut self, expected: &Token, err: impl FnOnce(Span) -> ParsingError, - ) -> super::Result> { + ) -> Result> { match self.next() { Some(Ok(tok)) if expected == &tok => Ok(tok), Some(Ok(_)) => Err(err(self.span())), @@ -61,7 +63,7 @@ impl<'a> Lexer<'a> { &mut self, expected: impl FnOnce(&Token<'a>) -> bool, msg: String, - ) -> super::Result> { + ) -> Result> { match self.next() { Some(Ok(tok)) if expected(&tok) => Ok(tok), Some(Ok(_)) => Err(ParsingError::UnexpectedToken(self.span(), msg)), @@ -70,7 +72,7 @@ impl<'a> Lexer<'a> { } } - pub fn expect_identifier(&mut self) -> super::Result> { + pub fn expect_identifier(&mut self) -> Result> { if let Some(Token::Identifier(ident)) = self.next_if(|t| matches!(t, Token::Identifier(_)))? { @@ -83,6 +85,28 @@ impl<'a> Lexer<'a> { } } + pub fn expect_string(&mut self) -> Result> { + if let Some(Token::String(string)) = self.next_if(|t| matches!(t, Token::String(_)))? { + Ok(string) + } else { + Err(ParsingError::UnexpectedToken( + self.peeked_span().unwrap_or(self.span()), + "expected a string".into(), + )) + } + } + + pub fn lex_ident(&self, source: &[u8], arena: &'a Bump) -> Result<&'a str> { + let Some(Ok(Token::Identifier(ident))) = Token::lex(source, arena, false, true).next() + else { + return Err(ParsingError::UnexpectedToken( + self.span().start + 1..self.span().end - 1, + "expected a valid, non-qualified identifier.".into(), + )); + }; + Ok(arena.alloc_str(ident.literal)) + } + pub fn consume(&mut self, token: &Token) -> bool { if let Some(Ok(next)) = self.peek() && next == token @@ -113,7 +137,7 @@ impl<'a> Lexer<'a> { self.advance_span(next).transpose() } - pub fn expect_next(&mut self) -> super::Result> { + pub fn expect_next(&mut self) -> Result> { match self.next() { None => Err(ParsingError::LexingError(LexingError::UnexpectedEof)), Some(Ok(tok)) => Ok(tok), @@ -121,7 +145,7 @@ impl<'a> Lexer<'a> { } } - pub fn expect_peek(&mut self) -> super::Result<&Token<'a>> { + pub fn expect_peek(&mut self) -> Result<&Token<'a>> { match self.peek() { None => Err(ParsingError::LexingError(LexingError::UnexpectedEof)), Some(Ok(tok)) => Ok(tok), @@ -133,14 +157,14 @@ impl<'a> Lexer<'a> { self.span.clone() } - pub fn peeked_span(&mut self) -> super::Result { + pub fn peeked_span(&mut self) -> Result { self.inner .peek() .map(|(_, s)| s.clone()) .ok_or(ParsingError::LexingError(LexingError::UnexpectedEof)) } - pub fn peek_with_span(&mut self) -> Option<(super::Result<&Token<'a>>, Span)> { + pub fn peek_with_span(&mut self) -> Option<(Result<&Token<'a>>, Span)> { self.inner.peek().map(|(a, b)| { ( a.as_ref().map_err(|e| ParsingError::LexingError(e.clone())), @@ -202,8 +226,10 @@ impl TokenExt for Token<'_> { fn is_expr_start(&self) -> bool { self.is_atom() || self.is_prefix_op() - || self == &Token::OpenParent - || self == &Token::Getline + || matches!( + self, + Token::IndirectCall(_) | Token::Getline | Token::OpenParent + ) } fn is_place(&self) -> bool { matches!( diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 5d1490f..53dddd3 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -118,26 +118,30 @@ impl<'a> Parser<'a> { }); } else { match lex.expect_next()? { - Token::LoadDirective(lib) => { + Token::LoadDirective => { + let lib = lex.expect_string()?; self.ast.loads.push(lib); lex.expect_with(Token::is_stmnt_end, "expected statement end.".into())?; } - Token::IncludeDirective(path) => { + Token::IncludeDirective => { + let path = lex.expect_string()?; let old_namespace = self.namespace; let content = self.preprocessor.include_in(path.as_ref(), self.arena); self.parse_top(&mut Lexer::new(content, self.arena), true)?; lex.expect_with(Token::is_stmnt_end, "expected statement end.".into())?; self.namespace = old_namespace; } - Token::NsIncludeDirective(path) => { + Token::NsIncludeDirective => { + let path = lex.expect_string()?; let old_namespace = self.namespace; let content = self.preprocessor.include_in(path.as_ref(), self.arena); self.parse_top(&mut Lexer::new(content, self.arena), false)?; lex.expect_with(Token::is_stmnt_end, "expected statement end.".into())?; self.namespace = old_namespace; } - Token::NamespaceDirective(namespace) => { - self.namespace = namespace; + Token::NamespaceDirective => { + let namespace = lex.expect_string()?; + self.namespace = lex.lex_ident(namespace.as_ref(), self.arena)?; lex.expect_with(Token::is_stmnt_end, "expected statement end.".into())?; } Token::ConcurrentDirective => { @@ -551,7 +555,7 @@ impl<'a> Parser<'a> { fn parse_delete(&mut self, lex: &mut Lexer<'a>) -> Result> { let next = lex.expect_next()?; - let Some(var) = self.get_place(lex, next) else { + let Ok(var) = self.get_place(lex, next) else { return Err(ParsingError::OperatorExpectsVariable(lex.span())); }; let index = if lex.consume(&Token::OpenBracket) { @@ -603,9 +607,10 @@ impl<'a> Parser<'a> { args.push(name); if !lex.consume(&Token::Comma) { - lex.expect(&Token::ClosedParent, |s| { - ParsingError::FunctionCallMissingParenthesis(s, name.to_string()) - })?; + lex.expect( + &Token::ClosedParent, + ParsingError::FunctionCallMissingParenthesis, + )?; break; } } @@ -628,21 +633,21 @@ impl<'a> Parser<'a> { .push(rule); } - #[tracing::instrument] fn parse_function_call( &mut self, lex: &mut Lexer<'a>, - name: Identifier<'a>, + generate: impl FnOnce(Vec<'a, Expr<'a>>) -> ExprNode<'a>, span: Span, ) -> Result> { lex.expect(&Token::OpenParent, ParsingError::ExpectedOpeningParenthesis)?; if lex.span().start != span.end { return Err(ParsingError::FunctionCallSeparatedIdent(span)); } - let expr = ExprNode::FunctionCall(name, self.parse_function_args(lex)?); - lex.expect(&Token::ClosedParent, |s| { - ParsingError::FunctionCallMissingParenthesis(s, name.to_string()) - })?; + let expr = generate(self.parse_function_args(lex)?); + lex.expect( + &Token::ClosedParent, + ParsingError::FunctionCallMissingParenthesis, + )?; Ok(Expr::node(expr, self.arena)) } @@ -660,8 +665,8 @@ impl<'a> Parser<'a> { Token::TypedRegex(r) if typed_regex => Ok(Atom::TypedRegex(r)), Token::TypedRegex(_) => Err(ParsingError::UnexpectedTypedRegex(lex.span())), token => match self.get_place(lex, token) { - Some(var) => Ok(Atom::Variable(var)), - None => Err(ParsingError::UnexpectedToken( + Ok(var) => Ok(Atom::Variable(var)), + Err(_) => Err(ParsingError::UnexpectedToken( lex.span(), "is not valid data.".into(), )), @@ -670,27 +675,27 @@ impl<'a> Parser<'a> { } #[tracing::instrument] - fn get_place(&self, lex: &mut Lexer<'a>, token: Token<'a>) -> Option> { + fn get_place(&self, lex: &mut Lexer<'a>, token: Token<'a>) -> Result, Token<'a>> { match token { Token::Identifier(a) if !(lex.peek_is(&Token::OpenParent) && lex.is_yuxtaposed()) => { - Some(a.qualify(self.namespace).into()) + Ok(a.qualify(self.namespace).into()) } - Token::NrVariable => Some(Variable::Nr), - Token::NfVariable => Some(Variable::Nf), - Token::FsVariable => Some(Variable::Fs), - Token::RsVariable => Some(Variable::Rs), - Token::OfsVariable => Some(Variable::Ofs), - Token::OrsVariable => Some(Variable::Ors), - Token::FilenameVariable => Some(Variable::Filename), - Token::ArgcVariable => Some(Variable::Argc), - Token::ArgvVariable => Some(Variable::Argv), - Token::SubsepVariable => Some(Variable::Subsep), - Token::FnrVariable => Some(Variable::Fnr), - Token::OfmtVariable => Some(Variable::Ofmt), - Token::RstartVariable => Some(Variable::Rstart), - Token::RlengthVariable => Some(Variable::Rlength), - Token::EnvironVariable => Some(Variable::Environ), - _ => None, + Token::NrVariable => Ok(Variable::Nr), + Token::NfVariable => Ok(Variable::Nf), + Token::FsVariable => Ok(Variable::Fs), + Token::RsVariable => Ok(Variable::Rs), + Token::OfsVariable => Ok(Variable::Ofs), + Token::OrsVariable => Ok(Variable::Ors), + Token::FilenameVariable => Ok(Variable::Filename), + Token::ArgcVariable => Ok(Variable::Argc), + Token::ArgvVariable => Ok(Variable::Argv), + Token::SubsepVariable => Ok(Variable::Subsep), + Token::FnrVariable => Ok(Variable::Fnr), + Token::OfmtVariable => Ok(Variable::Ofmt), + Token::RstartVariable => Ok(Variable::Rstart), + Token::RlengthVariable => Ok(Variable::Rlength), + Token::EnvironVariable => Ok(Variable::Environ), + tok => Err(tok), } } } diff --git a/parser/src/pratt.rs b/parser/src/pratt.rs index 432671e..22979e7 100644 --- a/parser/src/pratt.rs +++ b/parser/src/pratt.rs @@ -177,8 +177,38 @@ impl<'a, 'b> Pratt<'a, 'b> { && lex.peek_is(&Token::OpenParent) && lex.is_yuxtaposed() { - self.parser - .parse_function_call(lex, name.qualify(self.parser.namespace), lex.span()) + if name.namespace.is_none_or(|n| n == "awk") && is_special_var(name.literal) { + return Err(ParsingError::SpecialVariableCall( + lex.span(), + name.literal.to_string(), + )); + } + self.parser.parse_function_call( + lex, + |args| ExprNode::FunctionCall(name.qualify(self.parser.namespace), args), + lex.span(), + ) + } else if let Token::IndirectCall(name) = next { + // Possible gawk bug: it accepts special variables if qualified, + // even if it is with the `awk` namespace. + if name.namespace.is_none() && is_special_var(name.literal) { + return Err(ParsingError::SpecialVariableIndirectCall( + lex.span(), + name.literal.to_string(), + )); + } + let name = Variable::User(name.qualify(self.parser.namespace)); + self.parser.parse_function_call( + lex, + |args| ExprNode::IndirectCall(name, args), + lex.span(), + ) + } else if next.is_place() && lex.peek_is(&Token::OpenParent) && lex.is_yuxtaposed() { + let name = match self.parser.get_place(lex, next) { + Ok(var) => var.to_string(), + Err(tok) => format!("{tok:?}"), + }; + Err(ParsingError::SpecialVariableCall(lex.span(), name)) } else { match self.parser.parse_atom(lex, next, self.typed_regex) { Ok(atom) => Ok(Expr::leaf(atom)), @@ -321,3 +351,24 @@ impl<'a, 'b> Pratt<'a, 'b> { } } } + +fn is_special_var(name: &str) -> bool { + matches!( + name, + "NR" | "NF" + | "FS" + | "RS" + | "OFS" + | "ORS" + | "FILENAME" + | "ARGC" + | "ARGV" + | "SUBSEP" + | "FNR" + | "ARGIND" + | "OFMT" + | "RSTART" + | "RLENGTH" + | "ENVIRON" + ) +}