From b24337910bc92cdca8524c21bff03c895685ea15 Mon Sep 17 00:00:00 2001 From: "Guillem L. Jara" <4lon3ly0@tutanota.com> Date: Thu, 7 May 2026 16:51:05 +0200 Subject: [PATCH] lexer: POSIX and GNU compatibility modes Also improves ignored newlines on specific tokens. --- lexer/src/lib.rs | 171 +++++++++++++++++++++++++++++--------- parser/src/diagnostics.rs | 8 ++ parser/src/lex.rs | 3 +- 3 files changed, 144 insertions(+), 38 deletions(-) diff --git a/lexer/src/lib.rs b/lexer/src/lib.rs index 4fa506d..28c110e 100644 --- a/lexer/src/lib.rs +++ b/lexer/src/lib.rs @@ -25,12 +25,13 @@ pub type Result = std::result::Result; // TODO: check wnat GNU does about potentially reserved `@ident`; add error branch if so. #[derive(Logos, Debug, PartialEq)] #[logos(utf8 = false)] -#[logos(skip("[ \t]+"))] -#[logos(skip(r"(\\\n)+"))] +#[logos(skip(r"(?&ignore)"))] #[logos(skip("#", skip_line))] #[logos(extras = Extra)] #[logos(subpattern identifier = r"[a-zA-Z_][a-zA-Z0-9_]*")] -#[logos(error(LexingError, callback = |lex| LexingError::from(lex)))] +#[logos(subpattern ignore = r"(?:[ \t]|(\\\n))+")] +#[logos(subpattern ignore_with_nl = r"(?:(?&ignore)|\n)*")] +#[logos(error(LexingError, callback = |lex| LexingError::unexpected(lex)))] pub enum Token<'a> { #[regex(r"[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?", parse_float)] #[regex(r"\.[0-9]+([eE][+-]?[0-9]+)?", parse_float)] @@ -41,23 +42,23 @@ pub enum Token<'a> { BeginPattern, #[token("END", accept_expression)] EndPattern, - #[token("BEGINFILE", accept_expression)] + #[token("BEGINFILE", |lex| parse_non_posix_keyword(lex, Token::BeginFilePattern))] BeginFilePattern, - #[token("ENDFILE", accept_expression)] + #[token("ENDFILE", |lex| parse_non_posix_keyword(lex, Token::EndFilePattern))] EndFilePattern, #[token("@load \"", parse_directive)] LoadDirective(Slice<'a>), #[token("@include \"", parse_directive)] IncludeDirective(Slice<'a>), - #[token("@nsinclude \"", parse_directive)] + #[token("@nsinclude \"", parse_non_posix_directive)] NsIncludeDirective(Slice<'a>), - #[regex("@namespace \"(?&identifier)\"", |lex| parse_ident(lex, 12..lex.slice().len() - 1))] + #[regex("@namespace \"(?&identifier)\"", parse_namespace_directive)] NamespaceDirective(&'a str), - #[token("@concurrent", accept_expression)] + #[token("@concurrent", parse_non_gnu_operator)] ConcurrentDirective, #[token("if", accept_expression)] If, - #[regex(r"else\n*", accept_expression)] + #[regex(r"else(?&ignore_with_nl)", accept_expression)] Else, #[token("switch", accept_expression)] Switch, @@ -65,7 +66,7 @@ pub enum Token<'a> { Case, #[token("default", accept_expression)] Default, - #[regex(r"do\n*", accept_expression)] + #[regex(r"do(?&ignore_with_nl)", accept_expression)] Do, #[token("while", accept_expression)] While, @@ -94,6 +95,7 @@ pub enum Token<'a> { #[token("delete", accept_expression)] Delete, #[token("function", accept_expression)] + #[token("func", |lex| parse_non_posix_keyword(lex, Token::Function))] Function, #[token("NR", accept_expression)] NrVariable, @@ -143,6 +145,7 @@ pub enum Token<'a> { #[token("%", accept_expression)] Percent, #[token("^", accept_expression)] + #[token("**", parse_non_posix_operator)] Circumflex, #[token("++", accept_expression)] Increment, @@ -161,6 +164,7 @@ pub enum Token<'a> { #[token("%=", accept_expression)] PercentAssign, #[token("^=", accept_expression)] + #[token("**=", parse_non_posix_operator)] CaretAssign, #[token("==", accept_expression)] EqualTo, @@ -174,9 +178,9 @@ pub enum Token<'a> { GreaterThan, #[token(">=", accept_expression)] GreaterOrEqualThan, - #[regex(r"&&\n*", accept_expression)] + #[regex(r"&&(?&ignore_with_nl)", accept_expression)] BooleanAnd, - #[regex(r"\|\|\n*", accept_expression)] + #[regex(r"\|\|(?&ignore_with_nl)", accept_expression)] BooleanOr, #[token("!", accept_expression)] Negation, @@ -186,15 +190,17 @@ pub enum Token<'a> { NotMatching, #[token("|", accept_expression)] Pipe, - #[token("|&", accept_expression)] + #[token("|&", parse_non_posix_operator)] DoublePipe, #[token(">>", accept_expression)] AppendPipe, - #[regex(r"\?\n*", accept_expression)] + #[token("?", priority = 10)] + #[regex(r"\?(?&ignore_with_nl)", parse_non_posix_operator, priority = 11)] QuestionMark, - #[regex(r":\n*", accept_expression)] + #[token(":", priority = 10)] + #[regex(r":(?&ignore_with_nl)", parse_non_posix_operator, priority = 11)] Colon, - #[regex(r"\{\n*", accept_expression)] + #[regex(r"\{(?&ignore_with_nl)", accept_expression)] OpenBrace, #[token("}", accept_expression)] ClosedBrace, @@ -206,7 +212,7 @@ pub enum Token<'a> { OpenBracket, #[token("]", accept_operator)] ClosedBracket, - #[regex(r",\n*", accept_expression)] + #[regex(r",(?&ignore_with_nl)", accept_expression)] Comma, #[token("$", accept_expression)] Record, @@ -217,7 +223,12 @@ pub enum Token<'a> { } #[derive(Debug, Default, PartialEq, Eq)] -pub struct Extra(Context, *const Bump); +pub struct Extra { + ctx: Context, + arena: *const Bump, + posix_strict: bool, + gnu_strict: bool, +} #[derive(Debug, Default, PartialEq, Eq)] pub enum Context { @@ -231,7 +242,7 @@ pub enum LexingError { #[default] #[error("Unknown error")] Unknown, - #[error("Unexpected token at {:?}: `{}`", .0, .1)] + #[error("Unexpected token at {:?}: {:?}", .0, .1)] Unexpected(Span, String), #[error("Unterminated string at {:?}", .0)] UnterminatedString(Span), @@ -239,6 +250,10 @@ pub enum LexingError { UnterminatedRegex(Span), #[error("Unexpected End of File!")] UnexpectedEof, + #[error("Unknown token: {:?}.", .1)] + UnavailableOnPosix(Span, String), + #[error("Unknown token: {:?}.", .1)] + UnavailableOnGnu(Span, String), } #[derive(Debug, PartialEq, Eq, Clone, Copy)] @@ -248,14 +263,42 @@ pub struct Identifier<'a> { } impl<'a> Token<'a> { - pub fn lex(source: &'a [u8], arena: &'a Bump) -> logos::Lexer<'a, Self> { - Lexer::with_extras(source, Extra(Context::AcceptExpression, arena)) + pub fn lex( + source: &'a [u8], + arena: &'a Bump, + posix_strict: bool, + gnu_strict: bool, + ) -> logos::Lexer<'a, Self> { + Lexer::with_extras( + source, + Extra { + ctx: Context::AcceptExpression, + arena, + posix_strict, + gnu_strict, + }, + ) } } -impl From<&mut Lexer<'_>> for LexingError { - fn from(lex: &mut Lexer<'_>) -> Self { - Self::Unexpected(lex.span(), String::from_utf8_lossy(lex.slice()).to_string()) +impl LexingError { + fn to_utf8(lex: &mut Lexer<'_>) -> String { + String::from_utf8_lossy(lex.slice()).to_string() + } + + #[cold] + fn unexpected(lex: &mut Lexer<'_>) -> Self { + Self::Unexpected(lex.span(), Self::to_utf8(lex)) + } + + #[cold] + fn non_posix(lex: &mut Lexer<'_>) -> Self { + Self::UnavailableOnPosix(lex.span(), Self::to_utf8(lex)) + } + + #[cold] + fn non_uu(lex: &mut Lexer<'_>) -> Self { + Self::UnavailableOnGnu(lex.span(), Self::to_utf8(lex)) } } @@ -269,7 +312,7 @@ fn parse_string<'a>(lex: &mut logos::Lexer<'a, Token<'a>>) -> Result> } fn parse_regex_or_slash<'a>(lex: &mut logos::Lexer<'a, Token<'a>>) -> Result> { - match lex.extras.0 { + match lex.extras.ctx { Context::AcceptExpression => { accept_operator(lex); parse_content::(lex).map(Token::Regex) @@ -282,9 +325,28 @@ fn parse_regex_or_slash<'a>(lex: &mut logos::Lexer<'a, Token<'a>>) -> Result(lex: &mut Lexer<'a>) -> Result> { + accept_expression(lex); parse_content::(lex) } +fn parse_non_posix_directive<'a>(lex: &mut Lexer<'a>) -> Result> { + if lex.extras.posix_strict { + Err(LexingError::non_posix(lex)) + } else { + parse_directive(lex) + } +} + +fn parse_namespace_directive<'a>(lex: &mut Lexer<'a>) -> Result<&'a str> { + if lex.extras.posix_strict { + Err(LexingError::non_posix(lex)) + } else { + accept_expression(lex); + let offset = "@namespace \"".len(); + Ok(parse_ident(lex, offset..lex.slice().len() - 1)) + } +} + fn parse_content<'a, const MINIMAL: bool, const REGEX: bool, const DELIMITER: char>( lex: &mut Lexer<'a>, ) -> Result> { @@ -310,8 +372,11 @@ fn parse_content<'a, const MINIMAL: bool, const REGEX: bool, const DELIMITER: ch b'\\' => { out.to_mut(lex.extras.arena()) .extend_from_slice(&rest[start..i]); - let consumed = - parse_escape::(&rest[i..], out.to_mut(lex.extras.arena()))?; + let consumed = parse_escape::( + &rest[i..], + out.to_mut(lex.extras.arena()), + lex.extras.posix_strict, + )?; start = i + consumed; } _ => break, @@ -327,6 +392,7 @@ fn parse_content<'a, const MINIMAL: bool, const REGEX: bool, const DELIMITER: ch fn parse_escape( slice: &[u8], out: &mut Vec, + posix_strict: bool, ) -> Result { let mut count = 2; let is_oct = |x: char| ('0'..'8').contains(&x); @@ -368,7 +434,7 @@ fn parse_escape( .take(count - 1) .fold(0, |acc, digit| acc * 8 + digit - b'0') as char } - 'x' => todo!(), + 'x' if !posix_strict => todo!(), 'u' => todo!(), // Unspecified by POSIX; we ditto GNU. c => c, // TODO: Output warning @@ -388,6 +454,33 @@ fn parse_float(lex: &mut Lexer<'_>) -> f64 { parse_ident(lex, ..).parse().unwrap_or(0.) } +fn parse_non_posix_keyword<'a>(lex: &mut Lexer<'a>, other: Token<'a>) -> Token<'a> { + if lex.extras.posix_strict { + Token::Identifier(Identifier::without_namespace(lex)) + } else { + accept_expression(lex); + other + } +} + +fn parse_non_posix_operator(lex: &mut Lexer<'_>) -> Result<()> { + if lex.extras.posix_strict { + Err(LexingError::non_posix(lex)) + } else { + accept_expression(lex); + Ok(()) + } +} + +fn parse_non_gnu_operator(lex: &mut Lexer<'_>) -> Result<()> { + if lex.extras.gnu_strict { + Err(LexingError::non_uu(lex)) + } else { + accept_expression(lex); + Ok(()) + } +} + impl<'a> Identifier<'a> { fn without_namespace(lex: &mut Lexer<'a>) -> Self { Self { @@ -396,22 +489,26 @@ impl<'a> Identifier<'a> { } } - fn with_namespace(lex: &mut Lexer<'a>) -> Self { - // SAFETY: The regex matching ensures it is present and well-formed. - let separator = unsafe { memchr(b':', lex.slice()).unwrap_unchecked() }; - Self { - namespace: Some(parse_ident(lex, ..separator)), - literal: parse_ident(lex, separator + 2..), + fn with_namespace(lex: &mut Lexer<'a>) -> Result { + if lex.extras.posix_strict { + Err(LexingError::non_posix(lex)) + } else { + // SAFETY: The regex matching ensures it is present and well-formed. + let separator = unsafe { memchr(b':', lex.slice()).unwrap_unchecked() }; + Ok(Self { + namespace: Some(parse_ident(lex, ..separator)), + literal: parse_ident(lex, separator + 2..), + }) } } } fn accept_expression(lex: &mut Lexer<'_>) { - lex.extras.0 = Context::AcceptExpression; + lex.extras.ctx = Context::AcceptExpression; } fn accept_operator(lex: &mut Lexer<'_>) { - lex.extras.0 = Context::AcceptOperator; + lex.extras.ctx = Context::AcceptOperator; } #[derive(PartialEq, Eq, PartialOrd, Ord, Clone)] @@ -469,6 +566,6 @@ impl Extra { fn arena<'a>(&self) -> &'a Bump { // SAFETY: lives for as long as self because it's the same lifetime as // the source being lexed; Logos just can't take lifetimes on extras. - unsafe { &*self.1 } + unsafe { &*self.arena } } } diff --git a/parser/src/diagnostics.rs b/parser/src/diagnostics.rs index a7a9709..08ecdea 100644 --- a/parser/src/diagnostics.rs +++ b/parser/src/diagnostics.rs @@ -78,6 +78,8 @@ impl ParsingError { LexingError::UnterminatedString(span) => Some(span.clone()), LexingError::UnterminatedRegex(span) => Some(span.clone()), LexingError::UnexpectedEof => None, + LexingError::UnavailableOnPosix(span, _) => Some(span.clone()), + LexingError::UnavailableOnGnu(span, _) => Some(span.clone()), }, Self::UnclosedScope(span) => Some(span.clone()), Self::UnexpectedToken(span, _) => Some(span.clone()), @@ -131,6 +133,12 @@ impl ParsingError { Self::MissingTernaryOr(_) => Some( "Ternaries select between to expression based on a condition, like `bool ? foo : bar`.", ), + Self::LexingError(LexingError::UnavailableOnPosix(_, _)) => { + Some("This item is not available in POSIX-strict or traditional modes.") + } + Self::LexingError(LexingError::UnavailableOnGnu(_, _)) => { + Some("This item is not available in GNU-strict mode.") + } _ => None, } } diff --git a/parser/src/lex.rs b/parser/src/lex.rs index 5595508..6430a92 100644 --- a/parser/src/lex.rs +++ b/parser/src/lex.rs @@ -24,7 +24,8 @@ type LexItem<'a> = as Iterator>::Item; impl<'a> Lexer<'a> { pub fn new(source: &'a [u8], arena: &'a Bump) -> Self { Self { - inner: Token::lex(source, arena).spanned().peekable(), + // TODO: wire in POSIX & GNU strict conformance. + inner: Token::lex(source, arena, false, false).spanned().peekable(), span: Span::default(), // source, }