uutils/awk
0
0
Fork 0
mirror of https://github.com/uutils/awk.git synced 2026-06-10 16:15:04 -07:00
Code Issues Packages Projects Releases Wiki Activity

parser, lexer: indirect call support

Browse Source 
Required a small refactoring.
This commit is contained in:
Guillem L. Jara
2026-05-12 22:38:57 +02:00
parent 8993ad2fca
commit ba9c443369
8 changed files with 210 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lexer/src/lib.rs
+28 -38
View File
@@ -47,14 +47,14 @@ pub enum Token<'a> {
BeginFilePattern,
#[token("ENDFILE", |lex| parse_non_posix_keyword(lex, Token::EndFilePattern))]
EndFilePattern,
#[token("@load \"", parse_directive)]
LoadDirective(Slice<'a>),
#[token("@include \"", parse_directive)]
IncludeDirective(Slice<'a>),
#[token("@nsinclude \"", parse_non_posix_directive)]
NsIncludeDirective(Slice<'a>),
#[regex("@namespace \"(?&identifier)\"", parse_namespace_directive)]
NamespaceDirective(&'a str),
#[token("@load")]
LoadDirective,
#[token("@include")]
IncludeDirective,
#[token("@nsinclude", parse_non_posix_operator)]
NsIncludeDirective,
#[regex("@namespace", parse_non_posix_operator)]
NamespaceDirective,
#[token("@concurrent", parse_non_gnu_operator)]
ConcurrentDirective,
#[token("if", accept_expression)]
@@ -130,9 +130,12 @@ pub enum Token<'a> {
RlengthVariable,
#[token("ENVIRON", accept_expression)]
EnvironVariable,
#[regex("(?&identifier)", Identifier::without_namespace)]
#[regex(r"(?&identifier)::(?&identifier)", Identifier::with_namespace)]
#[regex("(?&identifier)", Identifier::without_namespace::<0>)]
#[regex(r"(?&identifier)::(?&identifier)", Identifier::with_namespace::<0>)]
Identifier(Identifier<'a>),
#[regex("@(?&identifier)", parse_indirect_call::<false>)]
#[regex(r"@(?&identifier)::(?&identifier)", parse_indirect_call::<true>)]
IndirectCall(Identifier<'a>),
#[token("+", accept_expression)]
Plus,
#[token("-", accept_expression)]
@@ -333,29 +336,6 @@ fn parse_regex_or_op<'a>(lex: &mut logos::Lexer<'a, Token<'a>>) -> Result<Token<
}
}
fn parse_directive<'a>(lex: &mut Lexer<'a>) -> Result<Slice<'a>> {
accept_expression(lex);
parse_content::<false, '"'>(lex)
}
fn parse_non_posix_directive<'a>(lex: &mut Lexer<'a>) -> Result<Slice<'a>> {
if lex.extras.posix_strict {
Err(LexingError::non_posix(lex))
} else {
parse_directive(lex)
}
}
fn parse_namespace_directive<'a>(lex: &mut Lexer<'a>) -> Result<&'a str> {
if lex.extras.posix_strict {
Err(LexingError::non_posix(lex))
} else {
accept_expression(lex);
let offset = "@namespace \"".len();
Ok(parse_ident(lex, offset..lex.slice().len() - 1))
}
}
fn parse_content<'a, const REGEX: bool, const DELIMITER: char>(
lex: &mut Lexer<'a>,
) -> Result<Slice<'a>> {
@@ -456,7 +436,7 @@ fn parse_float(lex: &mut Lexer<'_>) -> f64 {
fn parse_non_posix_keyword<'a>(lex: &mut Lexer<'a>, other: Token<'a>) -> Token<'a> {
if lex.extras.posix_strict {
Token::Identifier(Identifier::without_namespace(lex))
Token::Identifier(Identifier::without_namespace::<0>(lex))
} else {
accept_expression(lex);
other
@@ -481,22 +461,32 @@ fn parse_non_gnu_operator(lex: &mut Lexer<'_>) -> Result<()> {
}
}
fn parse_indirect_call<'a, const QUALIFIED: bool>(lex: &mut Lexer<'a>) -> Result<Identifier<'a>> {
if lex.extras.posix_strict {
Err(LexingError::non_posix(lex))
} else if QUALIFIED {
Identifier::with_namespace::<1>(lex)
} else {
Ok(Identifier::without_namespace::<1>(lex))
}
}
impl<'a> Identifier<'a> {
fn without_namespace(lex: &mut Lexer<'a>) -> Self {
fn without_namespace<const SKIP: usize>(lex: &mut Lexer<'a>) -> Self {
Self {
namespace: None,
literal: parse_ident(lex, ..),
literal: parse_ident(lex, SKIP..),
}
}
fn with_namespace(lex: &mut Lexer<'a>) -> Result<Self> {
fn with_namespace<const SKIP: usize>(lex: &mut Lexer<'a>) -> Result<Self> {
if lex.extras.posix_strict {
Err(LexingError::non_posix(lex))
} else {
// SAFETY: The regex matching ensures it is present and well-formed.
let separator = unsafe { memchr(b':', lex.slice()).unwrap_unchecked() };
Ok(Self {
namespace: Some(parse_ident(lex, ..separator)),
namespace: Some(parse_ident(lex, SKIP..separator)),
literal: parse_ident(lex, separator + 2..),
})
}
lexer/src/tests.rs
+31 -5
View File
@@ -153,17 +153,42 @@ fn lexer_test_directive_escaping() {
assert_eq!(
&lex(str, &arena, false, false),
&[
Token::IncludeDirective(b"aa\"a\ta".into()),
Token::NsIncludeDirective(b"b\"\nb".into())
Token::IncludeDirective,
Token::String(b"aa\"a\ta".into()),
Token::NsIncludeDirective,
Token::String(b"b\"\nb".into())
]
);
}
#[test]
#[should_panic]
fn lexer_test_ident_rules_non_posix() {
let arena = Bump::new();
lex(b"@namespace \"1a\"; a::1a", &arena, false, false);
assert_eq!(
&lex(b"1a::a a::1a _a", &arena, false, false),
&[
Token::Number(1.),
Token::Identifier(Identifier {
namespace: Some("a"),
literal: "a"
}),
Token::Identifier(Identifier {
namespace: None,
literal: "a"
}),
Token::Colon,
Token::Colon,
Token::Number(1.),
Token::Identifier(Identifier {
namespace: None,
literal: "a"
}),
Token::Identifier(Identifier {
namespace: None,
literal: "_a"
})
]
);
}
#[test]
@@ -186,7 +211,8 @@ fn lexer_test_general_tokens() {
&lex(str, &arena, false, false),
&[
Token::Newline,
Token::LoadDirective(b"lib1.so.1".into()),
Token::LoadDirective,
Token::String(b"lib1.so.1".into()),
Token::Newline,
Token::BeginPattern,
Token::OpenBrace,
parser/src/ast.rs
+8
View File
@@ -93,6 +93,7 @@ pub type Pattern<'a> = Either<RulePattern<'a>, SpecialPattern>;
#[derive(Debug)]
pub enum ExprNode<'a> {
FunctionCall(Identifier<'a>, Vec<'a, Expr<'a>>),
IndirectCall(Variable<'a>, Vec<'a, Expr<'a>>),
UnaryOperation(UnaryOperator, Expr<'a>),
BinaryOperation(BinaryOperator, Expr<'a>, Expr<'a>),
UnaryPlaceOperation(UnaryPlaceOperator, Place<'a>),
@@ -556,6 +557,13 @@ impl Debug for Expr<'_> {
}
write!(f, ")")
}
ExprNode::IndirectCall(ident, args) => {
write!(f, "(@{ident:?}")?;
for arg in args {
write!(f, " {arg:?}")?;
}
write!(f, ")")
}
ExprNode::UnaryOperation(op, a) => write!(f, "({op:?} {a:?})"),
ExprNode::BinaryOperation(op, a, b) => write!(f, "({op:?} {a:?} {b:?})"),
ExprNode::BinaryPlaceOperation(op, a, b) => write!(f, "({op:?} {a:?} {b:?})"),
parser/src/diagnostics.rs
+9 -3
View File
@@ -53,8 +53,8 @@ pub enum ParsingError {
InvalidExpression(Span, String),
#[error("Missing alternate branch in ternary expression.")]
MissingTernaryOr(Span),
#[error("Missing closing parenthesis in function call to `{}`.", .1)]
FunctionCallMissingParenthesis(Span, String),
#[error("Missing closing parenthesis in function call.")]
FunctionCallMissingParenthesis(Span),
#[error("Functions calls must have their name yuxtaposed to the parenthesis `(`.")]
FunctionCallSeparatedIdent(Span),
#[error("Missing closing parenthesis `(` in function call to `{}`.", .1)]
@@ -69,6 +69,10 @@ pub enum ParsingError {
ExpectedPlaceOperator(Span),
#[error("Typed regular expressions not accepted in this position.")]
UnexpectedTypedRegex(Span),
#[error("Can't call non-function, special variable `{}`.", .1)]
SpecialVariableCall(Span, String),
#[error("Can't use special variable `{}` for indirect function call.", .1)]
SpecialVariableIndirectCall(Span, String),
}
impl ParsingError {
@@ -104,7 +108,7 @@ impl ParsingError {
Self::OperatorExpectsVariable(span) => Some(span.clone()),
Self::InvalidExpression(span, _) => Some(span.clone()),
Self::MissingTernaryOr(span) => Some(span.clone()),
Self::FunctionCallMissingParenthesis(span, _) => Some(span.clone()),
Self::FunctionCallMissingParenthesis(span) => Some(span.clone()),
Self::FunctionCallSeparatedIdent(span) => Some(span.clone()),
Self::FunctionCallUnclosed(span, _) => Some(span.clone()),
Self::ExpectedIdentifier(span) => Some(span.clone()),
@@ -112,6 +116,8 @@ impl ParsingError {
Self::ExpectedBinaryOperator(span) => Some(span.clone()),
Self::ExpectedPlaceOperator(span) => Some(span.clone()),
Self::UnexpectedTypedRegex(span) => Some(span.clone()),
Self::SpecialVariableCall(span, _) => Some(span.clone()),
Self::SpecialVariableIndirectCall(span, _) => Some(span.clone()),
}
}
fn hint(&self) -> Option<&'static str> {
parser/src/idempotency.rs
+5
View File
@@ -277,6 +277,11 @@ impl Display for ExprNode<'_> {
write_args(f, args, indent)?;
write!(f, ")")
}
Self::IndirectCall(var, args) => {
write!(f, "@{var}(")?;
write_args(f, args, indent)?;
write!(f, ")")
}
Self::UnaryOperation(op, x) => {
let bp = op.binding_power();
let child_w = encode(indent, bp.saturating_add(1));
parser/src/lex.rs
+36 -10
View File
@@ -6,13 +6,15 @@
use std::{fmt::Debug, iter::Peekable};
use bumpalo::Bump;
use lexer::{LexingError, Span, SpannedIter, Token};
use lexer::{Identifier, LexingError, Slice, Span, SpannedIter, Token};
use crate::{
ParsingError,
ast::{Command, SpecialPattern},
};
use super::Result;
pub struct Lexer<'a> {
inner: Peekable<SpannedIter<'a, Token<'a>>>,
span: Span,
@@ -48,7 +50,7 @@ impl<'a> Lexer<'a> {
&mut self,
expected: &Token,
err: impl FnOnce(Span) -> ParsingError,
) -> super::Result<Token<'a>> {
) -> Result<Token<'a>> {
match self.next() {
Some(Ok(tok)) if expected == &tok => Ok(tok),
Some(Ok(_)) => Err(err(self.span())),
@@ -61,7 +63,7 @@ impl<'a> Lexer<'a> {
&mut self,
expected: impl FnOnce(&Token<'a>) -> bool,
msg: String,
) -> super::Result<Token<'a>> {
) -> Result<Token<'a>> {
match self.next() {
Some(Ok(tok)) if expected(&tok) => Ok(tok),
Some(Ok(_)) => Err(ParsingError::UnexpectedToken(self.span(), msg)),
@@ -70,7 +72,7 @@ impl<'a> Lexer<'a> {
}
}
pub fn expect_identifier(&mut self) -> super::Result<lexer::Identifier<'a>> {
pub fn expect_identifier(&mut self) -> Result<Identifier<'a>> {
if let Some(Token::Identifier(ident)) =
self.next_if(|t| matches!(t, Token::Identifier(_)))?
{
@@ -83,6 +85,28 @@ impl<'a> Lexer<'a> {
}
}
pub fn expect_string(&mut self) -> Result<Slice<'a>> {
if let Some(Token::String(string)) = self.next_if(|t| matches!(t, Token::String(_)))? {
Ok(string)
} else {
Err(ParsingError::UnexpectedToken(
self.peeked_span().unwrap_or(self.span()),
"expected a string".into(),
))
}
}
pub fn lex_ident(&self, source: &[u8], arena: &'a Bump) -> Result<&'a str> {
let Some(Ok(Token::Identifier(ident))) = Token::lex(source, arena, false, true).next()
else {
return Err(ParsingError::UnexpectedToken(
self.span().start + 1..self.span().end - 1,
"expected a valid, non-qualified identifier.".into(),
));
};
Ok(arena.alloc_str(ident.literal))
}
pub fn consume(&mut self, token: &Token) -> bool {
if let Some(Ok(next)) = self.peek()
&& next == token
@@ -113,7 +137,7 @@ impl<'a> Lexer<'a> {
self.advance_span(next).transpose()
}
pub fn expect_next(&mut self) -> super::Result<Token<'a>> {
pub fn expect_next(&mut self) -> Result<Token<'a>> {
match self.next() {
None => Err(ParsingError::LexingError(LexingError::UnexpectedEof)),
Some(Ok(tok)) => Ok(tok),
@@ -121,7 +145,7 @@ impl<'a> Lexer<'a> {
}
}
pub fn expect_peek(&mut self) -> super::Result<&Token<'a>> {
pub fn expect_peek(&mut self) -> Result<&Token<'a>> {
match self.peek() {
None => Err(ParsingError::LexingError(LexingError::UnexpectedEof)),
Some(Ok(tok)) => Ok(tok),
@@ -133,14 +157,14 @@ impl<'a> Lexer<'a> {
self.span.clone()
}
pub fn peeked_span(&mut self) -> super::Result<Span> {
pub fn peeked_span(&mut self) -> Result<Span> {
self.inner
.peek()
.map(|(_, s)| s.clone())
.ok_or(ParsingError::LexingError(LexingError::UnexpectedEof))
}
pub fn peek_with_span(&mut self) -> Option<(super::Result<&Token<'a>>, Span)> {
pub fn peek_with_span(&mut self) -> Option<(Result<&Token<'a>>, Span)> {
self.inner.peek().map(|(a, b)| {
(
a.as_ref().map_err(|e| ParsingError::LexingError(e.clone())),
@@ -202,8 +226,10 @@ impl TokenExt for Token<'_> {
fn is_expr_start(&self) -> bool {
self.is_atom()
|| self.is_prefix_op()
|| self == &Token::OpenParent
|| self == &Token::Getline
|| matches!(
self,
Token::IndirectCall(_) | Token::Getline | Token::OpenParent
)
}
fn is_place(&self) -> bool {
matches!(
parser/src/lib.rs
+40 -35
View File
@@ -118,26 +118,30 @@ impl<'a> Parser<'a> {
});
} else {
match lex.expect_next()? {
Token::LoadDirective(lib) => {
Token::LoadDirective => {
let lib = lex.expect_string()?;
self.ast.loads.push(lib);
lex.expect_with(Token::is_stmnt_end, "expected statement end.".into())?;
}
Token::IncludeDirective(path) => {
Token::IncludeDirective => {
let path = lex.expect_string()?;
let old_namespace = self.namespace;
let content = self.preprocessor.include_in(path.as_ref(), self.arena);
self.parse_top(&mut Lexer::new(content, self.arena), true)?;
lex.expect_with(Token::is_stmnt_end, "expected statement end.".into())?;
self.namespace = old_namespace;
}
Token::NsIncludeDirective(path) => {
Token::NsIncludeDirective => {
let path = lex.expect_string()?;
let old_namespace = self.namespace;
let content = self.preprocessor.include_in(path.as_ref(), self.arena);
self.parse_top(&mut Lexer::new(content, self.arena), false)?;
lex.expect_with(Token::is_stmnt_end, "expected statement end.".into())?;
self.namespace = old_namespace;
}
Token::NamespaceDirective(namespace) => {
self.namespace = namespace;
Token::NamespaceDirective => {
let namespace = lex.expect_string()?;
self.namespace = lex.lex_ident(namespace.as_ref(), self.arena)?;
lex.expect_with(Token::is_stmnt_end, "expected statement end.".into())?;
}
Token::ConcurrentDirective => {
@@ -551,7 +555,7 @@ impl<'a> Parser<'a> {
fn parse_delete(&mut self, lex: &mut Lexer<'a>) -> Result<SimpleStatement<'a>> {
let next = lex.expect_next()?;
let Some(var) = self.get_place(lex, next) else {
let Ok(var) = self.get_place(lex, next) else {
return Err(ParsingError::OperatorExpectsVariable(lex.span()));
};
let index = if lex.consume(&Token::OpenBracket) {
@@ -603,9 +607,10 @@ impl<'a> Parser<'a> {
args.push(name);
if !lex.consume(&Token::Comma) {
lex.expect(&Token::ClosedParent, |s| {
ParsingError::FunctionCallMissingParenthesis(s, name.to_string())
})?;
lex.expect(
&Token::ClosedParent,
ParsingError::FunctionCallMissingParenthesis,
)?;
break;
}
}
@@ -628,21 +633,21 @@ impl<'a> Parser<'a> {
.push(rule);
}
#[tracing::instrument]
fn parse_function_call(
&mut self,
lex: &mut Lexer<'a>,
name: Identifier<'a>,
generate: impl FnOnce(Vec<'a, Expr<'a>>) -> ExprNode<'a>,
span: Span,
) -> Result<Expr<'a>> {
lex.expect(&Token::OpenParent, ParsingError::ExpectedOpeningParenthesis)?;
if lex.span().start != span.end {
return Err(ParsingError::FunctionCallSeparatedIdent(span));
}
let expr = ExprNode::FunctionCall(name, self.parse_function_args(lex)?);
lex.expect(&Token::ClosedParent, |s| {
ParsingError::FunctionCallMissingParenthesis(s, name.to_string())
})?;
let expr = generate(self.parse_function_args(lex)?);
lex.expect(
&Token::ClosedParent,
ParsingError::FunctionCallMissingParenthesis,
)?;
Ok(Expr::node(expr, self.arena))
}
@@ -660,8 +665,8 @@ impl<'a> Parser<'a> {
Token::TypedRegex(r) if typed_regex => Ok(Atom::TypedRegex(r)),
Token::TypedRegex(_) => Err(ParsingError::UnexpectedTypedRegex(lex.span())),
token => match self.get_place(lex, token) {
Some(var) => Ok(Atom::Variable(var)),
None => Err(ParsingError::UnexpectedToken(
Ok(var) => Ok(Atom::Variable(var)),
Err(_) => Err(ParsingError::UnexpectedToken(
lex.span(),
"is not valid data.".into(),
)),
@@ -670,27 +675,27 @@ impl<'a> Parser<'a> {
}
#[tracing::instrument]
fn get_place(&self, lex: &mut Lexer<'a>, token: Token<'a>) -> Option<Variable<'a>> {
fn get_place(&self, lex: &mut Lexer<'a>, token: Token<'a>) -> Result<Variable<'a>, Token<'a>> {
match token {
Token::Identifier(a) if !(lex.peek_is(&Token::OpenParent) && lex.is_yuxtaposed()) => {
Some(a.qualify(self.namespace).into())
Ok(a.qualify(self.namespace).into())
}
Token::NrVariable => Some(Variable::Nr),
Token::NfVariable => Some(Variable::Nf),
Token::FsVariable => Some(Variable::Fs),
Token::RsVariable => Some(Variable::Rs),
Token::OfsVariable => Some(Variable::Ofs),
Token::OrsVariable => Some(Variable::Ors),
Token::FilenameVariable => Some(Variable::Filename),
Token::ArgcVariable => Some(Variable::Argc),
Token::ArgvVariable => Some(Variable::Argv),
Token::SubsepVariable => Some(Variable::Subsep),
Token::FnrVariable => Some(Variable::Fnr),
Token::OfmtVariable => Some(Variable::Ofmt),
Token::RstartVariable => Some(Variable::Rstart),
Token::RlengthVariable => Some(Variable::Rlength),
Token::EnvironVariable => Some(Variable::Environ),
_ => None,
Token::NrVariable => Ok(Variable::Nr),
Token::NfVariable => Ok(Variable::Nf),
Token::FsVariable => Ok(Variable::Fs),
Token::RsVariable => Ok(Variable::Rs),
Token::OfsVariable => Ok(Variable::Ofs),
Token::OrsVariable => Ok(Variable::Ors),
Token::FilenameVariable => Ok(Variable::Filename),
Token::ArgcVariable => Ok(Variable::Argc),
Token::ArgvVariable => Ok(Variable::Argv),
Token::SubsepVariable => Ok(Variable::Subsep),
Token::FnrVariable => Ok(Variable::Fnr),
Token::OfmtVariable => Ok(Variable::Ofmt),
Token::RstartVariable => Ok(Variable::Rstart),
Token::RlengthVariable => Ok(Variable::Rlength),
Token::EnvironVariable => Ok(Variable::Environ),
tok => Err(tok),
}
}
}
parser/src/pratt.rs
+53 -2
View File
@@ -177,8 +177,38 @@ impl<'a, 'b> Pratt<'a, 'b> {
&& lex.peek_is(&Token::OpenParent)
&& lex.is_yuxtaposed()
{
self.parser
.parse_function_call(lex, name.qualify(self.parser.namespace), lex.span())
if name.namespace.is_none_or(|n| n == "awk") && is_special_var(name.literal) {
return Err(ParsingError::SpecialVariableCall(
lex.span(),
name.literal.to_string(),
));
}
self.parser.parse_function_call(
lex,
|args| ExprNode::FunctionCall(name.qualify(self.parser.namespace), args),
lex.span(),
)
} else if let Token::IndirectCall(name) = next {
// Possible gawk bug: it accepts special variables if qualified,
// even if it is with the `awk` namespace.
if name.namespace.is_none() && is_special_var(name.literal) {
return Err(ParsingError::SpecialVariableIndirectCall(
lex.span(),
name.literal.to_string(),
));
}
let name = Variable::User(name.qualify(self.parser.namespace));
self.parser.parse_function_call(
lex,
|args| ExprNode::IndirectCall(name, args),
lex.span(),
)
} else if next.is_place() && lex.peek_is(&Token::OpenParent) && lex.is_yuxtaposed() {
let name = match self.parser.get_place(lex, next) {
Ok(var) => var.to_string(),
Err(tok) => format!("{tok:?}"),
};
Err(ParsingError::SpecialVariableCall(lex.span(), name))
} else {
match self.parser.parse_atom(lex, next, self.typed_regex) {
Ok(atom) => Ok(Expr::leaf(atom)),
@@ -321,3 +351,24 @@ impl<'a, 'b> Pratt<'a, 'b> {
}
}
}
fn is_special_var(name: &str) -> bool {
matches!(
name,
"NR" | "NF"
| "FS"
| "RS"
| "OFS"
| "ORS"
| "FILENAME"
| "ARGC"
| "ARGV"
| "SUBSEP"
| "FNR"
| "ARGIND"
| "OFMT"
| "RSTART"
| "RLENGTH"
| "ENVIRON"
)
}