lexer, parser: add small integer atoms

Useful for immediate values in the interpreter; allows us to save on register loads and loads & stores in the constants table.

lexer/src/lib.rs

@@ -34,7 +34,9 @@ pub type Result<T, E = LexingError> = std::result::Result<T, E>;
 #[logos(subpattern ignore_with_nl = r"(?:(?&ignore)|\n)*")]
 #[logos(error(LexingError, callback = |lex| LexingError::unexpected(lex)))]
 pub enum Token<'a> {
-    #[regex(r"(-)?[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?", parse_float)]
+    #[regex("(-128|(-)?(12[0-7]|1[01][0-9]|[1-9]?[0-9]))", parse_i8, priority = 9)]
+    SmallInt(i8),
+    #[regex(r"(-)?[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?", parse_float, priority = 8)]
     #[regex(r"\.[0-9]+([eE][+-]?[0-9]+)?", parse_float)]
     Number(f64),
     #[token("\"", parse_string)]
@@ -482,6 +484,11 @@ fn parse_float(lex: &mut Lexer<'_>) -> f64 {
     parse_ident(lex, ..).parse().unwrap_or(0.)
 }
 
+fn parse_i8(lex: &mut Lexer<'_>) -> i8 {
+    // SAFETY: The regex matchin ensures it is well-formed and in [-128, 127].
+    unsafe { parse_ident(lex, ..).parse().unwrap_unchecked() }
+}
+
 fn parse_non_posix_keyword<'a>(lex: &mut Lexer<'a>, other: Token<'a>) -> Token<'a> {
     if lex.extras.posix_strict {
         Token::Identifier(Identifier::without_namespace::<0>(lex))

lexer/src/tests.rs

@@ -121,20 +121,25 @@ fn lexer_test_gnu_pattern() {
 }
 
 #[test]
-fn lexer_test_floats() {
+fn lexer_test_nums() {
     let arena = Bump::new();
-    let str = b"1 20. 0. .3 2e4 -3.e2 5e+1 2.1e-3";
+    let str = b"1 20. 0. .3 2e4 -3.e2 5e+1 2.1e-3 -129 -128 -0 127 128";
     assert_eq!(
         &lex(str, &arena, false, false),
         &[
-            Token::Number(1.),
+            Token::SmallInt(1),
             Token::Number(20.),
             Token::Number(0.),
             Token::Number(0.3),
             Token::Number(2e4),
             Token::Number(-3e2),
             Token::Number(5e1),
-            Token::Number(2.1e-3)
+            Token::Number(2.1e-3),
+            Token::Number(-129.),
+            Token::SmallInt(-128),
+            Token::SmallInt(0),
+            Token::SmallInt(127),
+            Token::Number(128.)
         ]
     );
 }
@@ -160,12 +165,12 @@ fn lexer_test_ident_rules_non_posix() {
     assert_eq!(
         &lex(b"1a::a a::1a _a", &arena, false, false),
         &[
-            Token::Number(1.),
+            Token::SmallInt(1),
             Token::Identifier(Identifier { namespace: Some("a"), literal: "a" }),
             Token::Identifier(Identifier { namespace: None, literal: "a" }),
             Token::Colon,
             Token::Colon,
-            Token::Number(1.),
+            Token::SmallInt(1),
             Token::Identifier(Identifier { namespace: None, literal: "a" }),
             Token::Identifier(Identifier { namespace: None, literal: "_a" })
         ]
@@ -200,7 +205,7 @@ fn lexer_test_general_tokens() {
             Token::Print,
             Token::Identifier(Identifier { namespace: None, literal: "a" }),
             Token::Plus,
-            Token::Number(1.),
+            Token::SmallInt(1),
             Token::ClosedBrace,
             Token::Newline,
             Token::Regex(b"2\\..*".into()),
@@ -209,7 +214,7 @@ fn lexer_test_general_tokens() {
             Token::EndPattern,
             Token::OpenBrace,
             Token::Record,
-            Token::Number(1.),
+            Token::SmallInt(1),
             Token::EqualTo,
             Token::Identifier(Identifier { namespace: Some("foo"), literal: "bar" }),
             Token::ClosedBrace,
@@ -222,14 +227,14 @@ fn lexer_test_general_tokens() {
 fn lexer_test_regex_ambiguity() {
     let arena = Bump::new();
     assert_eq!(
-        &lex(b"1/=1 a/=1", &arena, false, false),
+        &lex(b"1/=1. a/=1", &arena, false, false),
         &[
-            Token::Number(1.),
+            Token::SmallInt(1),
             Token::SlashAssign,
             Token::Number(1.),
             Token::Identifier(Identifier { namespace: None, literal: "a" }),
             Token::SlashAssign,
-            Token::Number(1.)
+            Token::SmallInt(1)
         ]
     );
 }

parser/src/ast.rs

@@ -35,6 +35,7 @@ pub struct Rule<'a> {
 pub enum Atom<'a> {
     Variable(Variable<'a>),
     String(Slice<'a>),
+    SmallInt(i8),
     Number(f64),
     BigInt(),
     BigFloat(),

parser/src/lex.rs

@@ -219,8 +219,10 @@ impl TokenExt for Token<'_> {
         )
     }
     fn is_atom(&self) -> bool {
-        matches!(self, Token::Number(_) | Token::String(_) | Token::Regex(_))
+        matches!(
-            || self.is_place() && self != &Token::Record
+            self,
+            Token::Number(_) | Token::SmallInt(_) | Token::String(_) | Token::Regex(_)
+        ) || self.is_place() && self != &Token::Record
     }
     fn is_expr_start(&self) -> bool {
         self.is_atom()

parser/src/lib.rs

@@ -636,6 +636,7 @@ impl<'a> Parser<'a> {
     ) -> Result<Atom<'a>> {
         match token {
             Token::Number(n) => Ok(Atom::Number(n)),
+            Token::SmallInt(n) => Ok(Atom::SmallInt(n)),
             Token::String(s) => Ok(Atom::String(s)),
             Token::Regex(r) => Ok(Atom::Regex(r)),
             Token::TypedRegex(r) if typed_regex => Ok(Atom::TypedRegex(r)),

parser/src/sexpr.rs

@@ -292,6 +292,7 @@ impl Debug for Atom<'_> {
             Self::Variable(var) => write!(f, "{var:?}"),
             Self::String(str) => write!(f, "{str:?}"),
             Self::Number(num) => write!(f, "{num}"),
+            Self::SmallInt(num) => write!(f, "{num}"),
             Self::Regex(rgx) => write!(f, "/{rgx}/"),
             Self::TypedRegex(rgx) => write!(f, "@/{rgx}/"),
             Self::BigInt() | Self::BigFloat() => unimplemented!(),