From 55b1fbd561576b0ba3abea5cba0dbc2fbadc2dfb Mon Sep 17 00:00:00 2001 From: "Guillem L. Jara" <4lon3ly0@tutanota.com> Date: Fri, 29 May 2026 03:17:47 +0200 Subject: [PATCH] lexer, parser: add small integer atoms Useful for immediate values in the interpreter; allows us to save on register loads and loads & stores in the constants table. --- lexer/src/lib.rs | 9 ++++++++- lexer/src/tests.rs | 27 ++++++++++++++++----------- parser/src/ast.rs | 1 + parser/src/lex.rs | 6 ++++-- parser/src/lib.rs | 1 + parser/src/sexpr.rs | 1 + 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/lexer/src/lib.rs b/lexer/src/lib.rs index 950ff43..029ef9c 100644 --- a/lexer/src/lib.rs +++ b/lexer/src/lib.rs @@ -34,7 +34,9 @@ pub type Result = std::result::Result; #[logos(subpattern ignore_with_nl = r"(?:(?&ignore)|\n)*")] #[logos(error(LexingError, callback = |lex| LexingError::unexpected(lex)))] pub enum Token<'a> { - #[regex(r"(-)?[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?", parse_float)] + #[regex("(-128|(-)?(12[0-7]|1[01][0-9]|[1-9]?[0-9]))", parse_i8, priority = 9)] + SmallInt(i8), + #[regex(r"(-)?[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?", parse_float, priority = 8)] #[regex(r"\.[0-9]+([eE][+-]?[0-9]+)?", parse_float)] Number(f64), #[token("\"", parse_string)] @@ -482,6 +484,11 @@ fn parse_float(lex: &mut Lexer<'_>) -> f64 { parse_ident(lex, ..).parse().unwrap_or(0.) } +fn parse_i8(lex: &mut Lexer<'_>) -> i8 { + // SAFETY: The regex matchin ensures it is well-formed and in [-128, 127]. + unsafe { parse_ident(lex, ..).parse().unwrap_unchecked() } +} + fn parse_non_posix_keyword<'a>(lex: &mut Lexer<'a>, other: Token<'a>) -> Token<'a> { if lex.extras.posix_strict { Token::Identifier(Identifier::without_namespace::<0>(lex)) diff --git a/lexer/src/tests.rs b/lexer/src/tests.rs index f62e514..5ddabb0 100644 --- a/lexer/src/tests.rs +++ b/lexer/src/tests.rs @@ -121,20 +121,25 @@ fn lexer_test_gnu_pattern() { } #[test] -fn lexer_test_floats() { +fn lexer_test_nums() { let arena = Bump::new(); - let str = b"1 20. 0. .3 2e4 -3.e2 5e+1 2.1e-3"; + let str = b"1 20. 0. .3 2e4 -3.e2 5e+1 2.1e-3 -129 -128 -0 127 128"; assert_eq!( &lex(str, &arena, false, false), &[ - Token::Number(1.), + Token::SmallInt(1), Token::Number(20.), Token::Number(0.), Token::Number(0.3), Token::Number(2e4), Token::Number(-3e2), Token::Number(5e1), - Token::Number(2.1e-3) + Token::Number(2.1e-3), + Token::Number(-129.), + Token::SmallInt(-128), + Token::SmallInt(0), + Token::SmallInt(127), + Token::Number(128.) ] ); } @@ -160,12 +165,12 @@ fn lexer_test_ident_rules_non_posix() { assert_eq!( &lex(b"1a::a a::1a _a", &arena, false, false), &[ - Token::Number(1.), + Token::SmallInt(1), Token::Identifier(Identifier { namespace: Some("a"), literal: "a" }), Token::Identifier(Identifier { namespace: None, literal: "a" }), Token::Colon, Token::Colon, - Token::Number(1.), + Token::SmallInt(1), Token::Identifier(Identifier { namespace: None, literal: "a" }), Token::Identifier(Identifier { namespace: None, literal: "_a" }) ] @@ -200,7 +205,7 @@ fn lexer_test_general_tokens() { Token::Print, Token::Identifier(Identifier { namespace: None, literal: "a" }), Token::Plus, - Token::Number(1.), + Token::SmallInt(1), Token::ClosedBrace, Token::Newline, Token::Regex(b"2\\..*".into()), @@ -209,7 +214,7 @@ fn lexer_test_general_tokens() { Token::EndPattern, Token::OpenBrace, Token::Record, - Token::Number(1.), + Token::SmallInt(1), Token::EqualTo, Token::Identifier(Identifier { namespace: Some("foo"), literal: "bar" }), Token::ClosedBrace, @@ -222,14 +227,14 @@ fn lexer_test_general_tokens() { fn lexer_test_regex_ambiguity() { let arena = Bump::new(); assert_eq!( - &lex(b"1/=1 a/=1", &arena, false, false), + &lex(b"1/=1. a/=1", &arena, false, false), &[ - Token::Number(1.), + Token::SmallInt(1), Token::SlashAssign, Token::Number(1.), Token::Identifier(Identifier { namespace: None, literal: "a" }), Token::SlashAssign, - Token::Number(1.) + Token::SmallInt(1) ] ); } diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 7de8f1d..c0842f6 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -35,6 +35,7 @@ pub struct Rule<'a> { pub enum Atom<'a> { Variable(Variable<'a>), String(Slice<'a>), + SmallInt(i8), Number(f64), BigInt(), BigFloat(), diff --git a/parser/src/lex.rs b/parser/src/lex.rs index f2ab28b..dc63c19 100644 --- a/parser/src/lex.rs +++ b/parser/src/lex.rs @@ -219,8 +219,10 @@ impl TokenExt for Token<'_> { ) } fn is_atom(&self) -> bool { - matches!(self, Token::Number(_) | Token::String(_) | Token::Regex(_)) - || self.is_place() && self != &Token::Record + matches!( + self, + Token::Number(_) | Token::SmallInt(_) | Token::String(_) | Token::Regex(_) + ) || self.is_place() && self != &Token::Record } fn is_expr_start(&self) -> bool { self.is_atom() diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 0021068..5075939 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -636,6 +636,7 @@ impl<'a> Parser<'a> { ) -> Result> { match token { Token::Number(n) => Ok(Atom::Number(n)), + Token::SmallInt(n) => Ok(Atom::SmallInt(n)), Token::String(s) => Ok(Atom::String(s)), Token::Regex(r) => Ok(Atom::Regex(r)), Token::TypedRegex(r) if typed_regex => Ok(Atom::TypedRegex(r)), diff --git a/parser/src/sexpr.rs b/parser/src/sexpr.rs index 89d0777..dab1b94 100644 --- a/parser/src/sexpr.rs +++ b/parser/src/sexpr.rs @@ -292,6 +292,7 @@ impl Debug for Atom<'_> { Self::Variable(var) => write!(f, "{var:?}"), Self::String(str) => write!(f, "{str:?}"), Self::Number(num) => write!(f, "{num}"), + Self::SmallInt(num) => write!(f, "{num}"), Self::Regex(rgx) => write!(f, "/{rgx}/"), Self::TypedRegex(rgx) => write!(f, "@/{rgx}/"), Self::BigInt() | Self::BigFloat() => unimplemented!(),