interpreter: begin codegening expressions

Cargo.lock

@@ -502,7 +502,9 @@ version = "0.1.0"
 dependencies = [
  "color-eyre",
  "either",
+ "indexmap",
  "memchr",
+ "parser",
  "thiserror",
  "tracing",
 ]

interpreter/Cargo.toml

@@ -10,6 +10,8 @@ either.workspace = true
 memchr.workspace = true
 thiserror.workspace = true
 tracing.workspace = true
+parser.workspace = true
+indexmap = "2.14.0"
 
 [lints]
 workspace = true

interpreter/src/ir.rs

@@ -0,0 +1,386 @@
+// This file is part of the uutils awk package.
+//
+// For the full copyright and license information, please view the LICENSE
+// files that was distributed with this source code.
+
+//! This module contains the bytecode description, designed to be compact
+//! for cache efficiency and isomorphic w.r.t Cranelift IR. Also, our bytecode
+//! _is_ our IR; we lower the AST into it and can execute it right away, or do
+//! an optimization or JIT pass. We don't do the hack Lua 5's VM does of
+//! emitting bytecode without an intermediate AST because AWK contextual
+//! shenanigans; _even_ if it was possible, good luck maintaining that.
+
+#![allow(dead_code)]
+
+mod lower;
+
+use std::{
+    fmt::{Debug, Display},
+    ops::Deref,
+};
+
+pub use lower::test_interpreter;
+
+#[derive(Clone, Copy, Debug)]
+#[repr(transparent)]
+struct NonLocal(u16);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[repr(transparent)]
+struct Reg(u16);
+
+#[derive(Clone, Copy, Debug)]
+#[repr(transparent)]
+struct Label(u16);
+
+#[derive(Clone, Copy, Debug)]
+#[repr(transparent)]
+struct ArgCount(u16);
+
+#[repr(u8, align(1))]
+#[derive(Clone, Copy, Debug)]
+enum OpCode {
+    // Unary operations
+    Record,
+    Negation,
+    ToInt,
+    Negative,
+    Concat,
+
+    // Binary operations
+    Eq,
+    NEq,
+    Gt,
+    Lt,
+    LtE,
+    GtE,
+    And,
+    Or,
+    Matches,
+    MatchesNot,
+    Add,
+    Subtract,
+    Multiply,
+    Divide,
+    Raise,
+    Modulo,
+
+    // Intrinsic operations
+    Load,
+    LoadConst,
+    Copy,
+    Store,
+    IntrinsicCall,
+    UserCall,
+    IndirectCall,
+    Jump,
+    Return,
+    Branch,
+    BrIf,
+}
+
+const _: () = const { assert!(size_of::<Instruction>() <= 8) };
+
+#[derive(Clone, Copy)]
+#[repr(C, align(8))]
+struct Instruction {
+    opcode: OpCode,
+    // hint: Hint,
+    args: Arguments,
+}
+
+#[derive(Clone, Copy)]
+#[repr(C, align(2))]
+union Arguments {
+    unary_local: (Reg, Reg),
+    binary_local: (Reg, Reg, Reg),
+    load_store: (Reg, NonLocal),
+    jump: Label,
+    ret: Reg,
+    branch: (Reg, Label, Label),
+    br_if: (Reg, Label),
+    call: (Reg, NonLocal, ArgCount),
+    ind_call: (Reg, Reg, ArgCount),
+}
+
+impl Instruction {
+    fn unary(opcode: impl Into<OpCode>, dest: Reg, src: &impl Deref<Target = Reg>) -> Self {
+        let opcode = opcode.into();
+        debug_assert!(opcode.is_unary());
+        Self {
+            opcode,
+            args: Arguments {
+                unary_local: (dest, **src),
+            },
+        }
+    }
+
+    fn binary(
+        opcode: impl Into<OpCode>,
+        dest: Reg,
+        lhs: &impl Deref<Target = Reg>,
+        rhs: &impl Deref<Target = Reg>,
+    ) -> Self {
+        let opcode = opcode.into();
+        debug_assert!(opcode.is_binary());
+        Self {
+            opcode,
+            args: Arguments {
+                binary_local: (dest, **lhs, **rhs),
+            },
+        }
+    }
+
+    fn load_store(opcode: impl Into<OpCode>, dest: Reg, src: NonLocal) -> Self {
+        let opcode = opcode.into();
+        debug_assert!(opcode.is_load_store());
+        Self {
+            opcode,
+            args: Arguments {
+                load_store: (dest, src),
+            },
+        }
+    }
+
+    fn jump(opcode: impl Into<OpCode>, to: Label) -> Self {
+        let opcode = opcode.into();
+        debug_assert!(opcode.is_jump());
+        Self {
+            opcode,
+            args: Arguments { jump: to },
+        }
+    }
+
+    fn branch(opcode: impl Into<OpCode>, cond: Reg, true_to: Label, false_to: Label) -> Self {
+        let opcode = opcode.into();
+        debug_assert!(opcode.is_branch());
+        Self {
+            opcode,
+            args: Arguments {
+                branch: (cond, true_to, false_to),
+            },
+        }
+    }
+
+    fn br_if(opcode: impl Into<OpCode>, cond: Reg, to: Label) -> Self {
+        let opcode = opcode.into();
+        debug_assert!(opcode.is_branch_if());
+        Self {
+            opcode,
+            args: Arguments { br_if: (cond, to) },
+        }
+    }
+}
+
+impl OpCode {
+    fn is_unary(self) -> bool {
+        matches!(
+            self,
+            Self::Record | Self::Negation | Self::ToInt | Self::Negative | Self::Concat
+        )
+    }
+
+    fn is_binary(self) -> bool {
+        matches!(
+            self,
+            Self::Eq
+                | Self::NEq
+                | Self::Gt
+                | Self::Lt
+                | Self::LtE
+                | Self::GtE
+                | Self::And
+                | Self::Or
+                | Self::Matches
+                | Self::MatchesNot
+                | Self::Add
+                | Self::Subtract
+                | Self::Multiply
+                | Self::Divide
+                | Self::Raise
+                | Self::Modulo
+        )
+    }
+
+    fn is_load_store(self) -> bool {
+        matches!(self, Self::Load | Self::Store | Self::LoadConst)
+    }
+
+    fn is_jump(self) -> bool {
+        matches!(self, Self::Jump)
+    }
+
+    fn is_branch(self) -> bool {
+        matches!(self, Self::Branch)
+    }
+
+    fn is_branch_if(self) -> bool {
+        matches!(self, Self::BrIf)
+    }
+}
+
+// #[derive(Clone, Copy)]
+// #[repr(u8, align(1))]
+// enum Hint {
+//     // Next instruction contains additional data; for use in pipes, etc.
+//     // Also allows us to move past (2^16 - 1) variables if we want to.
+// }
+
+impl Debug for Instruction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Instruction::{:?}", self.opcode)?;
+        match self.opcode {
+            OpCode::Record
+            | OpCode::Negation
+            | OpCode::ToInt
+            | OpCode::Negative
+            | OpCode::Concat
+            | OpCode::Copy => {
+                let (dest, data) = unsafe { &self.args.unary_local };
+                write!(f, "({dest:?}, {data:?})")
+            }
+            OpCode::Eq
+            | OpCode::NEq
+            | OpCode::Gt
+            | OpCode::Lt
+            | OpCode::LtE
+            | OpCode::GtE
+            | OpCode::And
+            | OpCode::Or
+            | OpCode::Matches
+            | OpCode::MatchesNot
+            | OpCode::Add
+            | OpCode::Subtract
+            | OpCode::Multiply
+            | OpCode::Divide
+            | OpCode::Raise
+            | OpCode::Modulo => {
+                let (dest, lhs, rhs) = unsafe { &self.args.binary_local };
+                write!(f, "({dest:?}, {lhs:?}, {rhs:?})")
+            }
+            OpCode::Load | OpCode::LoadConst | OpCode::Store => {
+                let (dest, src) = unsafe { &self.args.load_store };
+                write!(f, "({dest:?}, {src:?})")
+            }
+            OpCode::BrIf => {
+                let (cond, label) = unsafe { self.args.br_if };
+                write!(f, "({cond:?}, {label:?})")
+            }
+            OpCode::Jump => {
+                let label = unsafe { self.args.jump };
+                write!(f, "({label:?})")
+            }
+            _ => todo!(),
+        }
+    }
+}
+
+impl Display for Instruction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self.opcode {
+            op @ (OpCode::Record
+            | OpCode::Negation
+            | OpCode::ToInt
+            | OpCode::Negative
+            | OpCode::Concat
+            | OpCode::Copy) => {
+                let (dest, data) = unsafe { &self.args.unary_local };
+                write!(f, "{dest} <- {op} {data}")
+            }
+            op @ (OpCode::Eq
+            | OpCode::NEq
+            | OpCode::Gt
+            | OpCode::Lt
+            | OpCode::LtE
+            | OpCode::GtE
+            | OpCode::And
+            | OpCode::Or
+            | OpCode::Matches
+            | OpCode::MatchesNot
+            | OpCode::Add
+            | OpCode::Subtract
+            | OpCode::Multiply
+            | OpCode::Divide
+            | OpCode::Raise
+            | OpCode::Modulo) => {
+                let (dest, lhs, rhs) = unsafe { &self.args.binary_local };
+                write!(f, "{dest} <- {op} {lhs}, {rhs}")
+            }
+            op @ (OpCode::Load | OpCode::Store) => {
+                let (dest, src) = unsafe { &self.args.load_store };
+                write!(f, "{dest} <- {op} global[{src}]")
+            }
+            op @ OpCode::LoadConst => {
+                let (dest, src) = unsafe { &self.args.load_store };
+                write!(f, "{dest} <- {op} mem[{src}]")
+            }
+            op @ OpCode::BrIf => {
+                let (cond, label) = unsafe { self.args.br_if };
+                write!(f, "{op} {cond}, {label}")
+            }
+            op @ OpCode::Jump => {
+                let label = unsafe { self.args.jump };
+                write!(f, "{op} {label}")
+            }
+            _ => todo!(),
+        }
+    }
+}
+
+impl Display for OpCode {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let str = match self {
+            Self::Record => "rec",
+            Self::Negation => "not",
+            Self::ToInt => "int",
+            Self::Negative => "neg",
+            Self::Concat => "cat",
+            Self::Eq => "eq",
+            Self::NEq => "neq",
+            Self::Gt => "gt",
+            Self::Lt => "lt",
+            Self::LtE => "le",
+            Self::GtE => "ge",
+            Self::And => "and",
+            Self::Or => "or",
+            Self::Matches => "mtch",
+            Self::MatchesNot => "nmtch",
+            Self::Add => "add",
+            Self::Subtract => "sub",
+            Self::Multiply => "mul",
+            Self::Divide => "div",
+            Self::Raise => "pow",
+            Self::Modulo => "mod",
+            Self::Load => "vload",
+            Self::LoadConst => "cload",
+            Self::Store => "vstore",
+            Self::Copy => "cpy",
+            Self::IntrinsicCall => "icall",
+            Self::UserCall => "ucall",
+            Self::IndirectCall => "vcall",
+            Self::Jump => "jmp",
+            Self::Return => "ret",
+            Self::Branch => "br",
+            Self::BrIf => "brif",
+        };
+        <_ as Display>::fmt(str, f)
+    }
+}
+
+impl Display for Label {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        <_ as Display>::fmt(&self.0, f)
+    }
+}
+
+impl Display for Reg {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "r{}", self.0)
+    }
+}
+
+impl Display for NonLocal {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}

interpreter/src/ir/lower.rs

@@ -0,0 +1,278 @@
+// This file is part of the uutils awk package.
+//
+// For the full copyright and license information, please view the LICENSE
+// files that was distributed with this source code.
+
+use std::fmt::Display;
+use std::hash::Hash;
+use std::mem::forget;
+use std::ops::Deref;
+
+use indexmap::IndexSet;
+use parser::{Atom, BinaryOperator, Expr, ExprNode, UnaryOperator};
+
+use crate::ir::{Instruction, Label, NonLocal, OpCode, Reg};
+
+#[derive(Clone, Copy, Debug, PartialEq, PartialOrd)]
+struct Value(f64); // TODO: use NaN-boxing.
+
+#[derive(Debug)]
+struct Code {
+    bc: Bytecode,
+    consts: IndexSet<Value>,
+    free_regs: Vec<Reg>,
+    reg_pointer: u16,
+}
+
+#[must_use]
+#[derive(Debug)]
+#[repr(transparent)]
+struct LinearReg(Reg);
+
+impl Code {
+    fn lower_expr(&mut self, expr: &Expr) -> LinearReg {
+        let dest = self.alloc_reg();
+        self.lower_expr_into(expr, dest);
+        LinearReg(dest)
+    }
+    fn lower_expr_into(&mut self, expr: &Expr, dest: Reg) {
+        match expr {
+            Expr::Leaf(atom) => match atom {
+                Atom::Variable(var) => {
+                    let src = NonLocal(crate::index_for_global(var));
+                    self.bc
+                        .emit(Instruction::load_store(OpCode::Load, dest, src));
+                }
+                &Atom::Number(n) => {
+                    let src = self.register_const(Value(n));
+                    self.bc
+                        .emit(Instruction::load_store(OpCode::LoadConst, dest, src));
+                }
+                _ => todo!(),
+            },
+            Expr::Node(node) => match node.as_ref() {
+                ExprNode::UnaryOperation(op, expr) => {
+                    let src = self.lower_expr(expr);
+                    self.bc.emit(Instruction::unary(*op, dest, &src));
+                    self.free_reg(src);
+                }
+                ExprNode::BinaryOperation(op, lhs, rhs) => {
+                    let lhs = self.lower_expr(lhs);
+                    let rhs = self.lower_expr(rhs);
+                    self.bc.emit(Instruction::binary(*op, dest, &lhs, &rhs));
+                    self.free_reg(lhs);
+                    self.free_reg(rhs);
+                }
+                ExprNode::Ternary(cond, true_then, false_then) => {
+                    self.lower_expr_into(cond, dest);
+
+                    let mut state = RegsState::new(self);
+                    let br_if = self
+                        .bc
+                        .emit(Instruction::br_if(OpCode::BrIf, dest, Label(0)));
+
+                    state = state.scope(self, |c| c.lower_expr_into(false_then, dest));
+
+                    let jump = self.bc.emit(Instruction::jump(OpCode::Jump, Label(0)));
+                    let label = Label(self.bc.len());
+
+                    state.scope_hwm(self, |c| c.lower_expr_into(true_then, dest));
+
+                    self.bc.nth(br_if).args.br_if.1 = label;
+                    self.bc.nth(jump).args.jump = Label(self.bc.len());
+                }
+                _ => todo!(),
+            },
+        }
+    }
+
+    fn alloc_reg(&mut self) -> Reg {
+        self.free_regs.pop().unwrap_or_else(|| {
+            let current = self.reg_pointer;
+            self.reg_pointer += 1;
+            Reg(current)
+        })
+    }
+
+    fn free_reg(&mut self, reg: LinearReg) {
+        self.free_regs.push(reg.into_inner());
+    }
+
+    fn register_const(&mut self, value: Value) -> NonLocal {
+        NonLocal(self.consts.insert_full(value).0 as u16)
+    }
+}
+
+#[derive(Debug, Default, Clone)]
+struct Bytecode {
+    code: Vec<Instruction>,
+}
+
+#[derive(Clone, Debug)]
+struct RegsState {
+    reg_pointer: u16,
+    n_free_regs: usize,
+}
+
+impl Bytecode {
+    fn new() -> Self {
+        Self::default()
+    }
+
+    #[inline(always)]
+    fn emit(&mut self, code: Instruction) -> Label {
+        self.code.push(code);
+        Label((self.code.len() - 1) as u16)
+    }
+
+    fn len(&self) -> u16 {
+        self.code.len() as u16
+    }
+
+    fn nth(&mut self, label: Label) -> &mut Instruction {
+        &mut self.code[label.0 as usize]
+    }
+}
+
+impl RegsState {
+    fn new(code: &Code) -> Self {
+        Self {
+            reg_pointer: code.reg_pointer,
+            n_free_regs: code.free_regs.len(),
+        }
+    }
+    fn scope(self, code: &mut Code, f: impl FnOnce(&mut Code)) -> Self {
+        f(code);
+        let old = code.reg_pointer;
+        code.reg_pointer = self.reg_pointer;
+        code.free_regs.truncate(self.n_free_regs);
+        Self {
+            reg_pointer: old,
+            n_free_regs: self.n_free_regs,
+        }
+    }
+    fn scope_hwm(self, code: &mut Code, f: impl FnOnce(&mut Code)) {
+        f(code);
+        code.reg_pointer = code.reg_pointer.max(self.reg_pointer);
+        code.free_regs.truncate(self.n_free_regs);
+    }
+}
+
+pub fn test_interpreter(expr: &Expr<'_>) -> impl Display {
+    let mut c = Code {
+        bc: Bytecode::new(),
+        consts: IndexSet::new(),
+        reg_pointer: 0,
+        free_regs: Vec::new(),
+    };
+    let result = c.lower_expr(expr);
+    dbg!(&c, &result);
+    forget(result);
+    c
+}
+
+impl From<UnaryOperator> for OpCode {
+    fn from(value: UnaryOperator) -> Self {
+        match value {
+            UnaryOperator::Record => Self::Record,
+            UnaryOperator::Negation => Self::Negation,
+            UnaryOperator::ToInt => Self::ToInt,
+            UnaryOperator::Negative => Self::Negative,
+        }
+    }
+}
+
+impl From<BinaryOperator> for OpCode {
+    fn from(value: BinaryOperator) -> Self {
+        match value {
+            BinaryOperator::Concat => Self::Concat,
+            BinaryOperator::Eq => Self::Eq,
+            BinaryOperator::NEq => Self::NEq,
+            BinaryOperator::Gt => Self::Gt,
+            BinaryOperator::Lt => Self::Lt,
+            BinaryOperator::LtE => Self::LtE,
+            BinaryOperator::GtE => Self::GtE,
+            BinaryOperator::And => Self::And,
+            BinaryOperator::Or => Self::Or,
+            BinaryOperator::Matches => Self::Matches,
+            BinaryOperator::MatchesNot => Self::MatchesNot,
+            BinaryOperator::Add => Self::Add,
+            BinaryOperator::Subtract => Self::Subtract,
+            BinaryOperator::Multiply => Self::Multiply,
+            BinaryOperator::Divide => Self::Divide,
+            BinaryOperator::Raise => Self::Raise,
+            BinaryOperator::Modulo => Self::Modulo,
+        }
+    }
+}
+
+impl Hash for Value {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        state.write(&self.0.to_ne_bytes());
+    }
+}
+
+impl Eq for Value {}
+
+// trait Fold<T> {
+//     type Args;
+//     fn fold(&self, args: Self::Args) -> T;
+// }
+
+impl Display for Bytecode {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let n = self.code.len() / 10 + 1;
+        for (i, e) in self.code.iter().enumerate() {
+            write!(f, "{i:n$}: {e}")?;
+            if i + 1 < self.code.len() as _ {
+                writeln!(f)?;
+            }
+        }
+        Ok(())
+    }
+}
+
+impl Display for Code {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(f, "Bytecode:\n{}\n", self.bc)?;
+        writeln!(f, "Consts:")?;
+        for (i, e) in self.consts.iter().enumerate() {
+            write!(f, "mem[{i}] = {}", e.0)?;
+            if i + 1 < self.consts.len() as _ {
+                writeln!(f)?;
+            }
+        }
+        Ok(())
+    }
+}
+
+impl LinearReg {
+    fn into_inner(self) -> Reg {
+        let inner = self.0;
+        forget(self);
+        inner
+    }
+}
+
+impl Deref for LinearReg {
+    type Target = Reg;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+// impl Deref for Reg {
+//     type Target = Self;
+
+//     fn deref(&self) -> &Self::Target {
+//         self
+//     }
+// }
+
+#[cfg(debug_assertions)]
+impl Drop for LinearReg {
+    fn drop(&mut self) {
+        debug_assert!(false, "Leaked register {}!", self.0);
+    }
+}

interpreter/src/lib.rs

@@ -3,14 +3,19 @@
 // For the full copyright and license information, please view the LICENSE
 // files that was distributed with this source code.
 
+mod ir;
+
 use color_eyre::eyre::Result;
 use either::Either;
 
+pub use ir::test_interpreter;
+use parser::Variable;
+
 pub enum BuiltinCommand {}
 pub enum BuiltinVar {}
 
 pub type Command<'a> = Either<BuiltinCommand, &'a str>;
-pub type Variable<'a> = Either<BuiltinVar, &'a str>;
+// pub type Variable<'a> = Either<BuiltinVar, &'a str>;
 
 #[derive(Debug)]
 pub struct Interpreter;
@@ -23,6 +28,10 @@ impl Interpreter {
     pub fn eval_expression(&mut self) {}
 }
 
+fn index_for_global(_var: &Variable) -> u16 {
+    1 // TODO
+}
+
 #[derive(Debug, thiserror::Error)]
 #[allow(dead_code)]
 enum InterpreterError {}

parser/src/ast.rs

@@ -149,7 +149,6 @@ pub enum BinaryPlaceOperator {
     PowAssign,
     ModAssign,
 }
-
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum ArrayOperator {
     Index,

parser/src/lib.rs

@@ -21,13 +21,9 @@ use either::Either::{Left, Right};
 use hashbrown::HashMap;
 use lexer::{LexingError, Span, Token};
 
-pub use crate::ast::Ast;
+pub use crate::ast::*;
 pub use crate::lex::Lexer;
 use crate::{
-    ast::{
-        ArrayOperator, Atom, Body, Command, Expr, ExprNode, Function, Identifier, Pattern,
-        Redirection, Rule, RulePattern, SimpleStatement, SpecialPattern, Statement, Variable,
-    },
     diagnostics::{ParsingError, report_error},
     lex::TokenExt,
     pratt::Pratt,

src/main.rs

@@ -14,7 +14,8 @@ use std::io::{self, Write};
 use bumpalo::Bump;
 use clap::Parser as _;
 use color_eyre::Result;
-use parser::Parser;
+use interpreter::test_interpreter;
+use parser::{Body, Parser, Rule};
 
 use crate::{
     cli::Args,
@@ -53,6 +54,20 @@ fn uu_main() -> Result<()> {
     }
     dbg!(arena.chunk_capacity());
 
+    if let Some(Rule {
+        actions: Some(Body(a)),
+        pattern: _,
+    }) = ast.rules.first()
+        && let Some(parser::Statement::Simple(parser::SimpleStatement::Expression(expr))) =
+            a.first()
+    {
+        let x = test_interpreter(expr);
+        if let Err(e) = writeln!(io::stdout(), "---\n{x}")
+            && e.kind() != io::ErrorKind::BrokenPipe
+        {
+            exit_err(Some(format!("awk: error writing to standard output: {e}")));
+        }
+    }
     // for token in lex {
     //     let Ok(x) = token else {
     //         return token.map(drop).map_err(color_eyre::Report::from);