From d82792d3ee5742c2865da99453fdf66f60924273 Mon Sep 17 00:00:00 2001 From: "Guillem L. Jara" <4lon3ly0@tutanota.com> Date: Thu, 21 May 2026 17:23:08 +0200 Subject: [PATCH] interpreter: begin codegening expressions --- Cargo.lock | 2 + interpreter/Cargo.toml | 2 + interpreter/src/ir.rs | 386 ++++++++++++++++++++++++++++++++++++ interpreter/src/ir/lower.rs | 278 ++++++++++++++++++++++++++ interpreter/src/lib.rs | 11 +- parser/src/ast.rs | 1 - parser/src/lib.rs | 6 +- src/main.rs | 17 +- 8 files changed, 695 insertions(+), 8 deletions(-) create mode 100644 interpreter/src/ir.rs create mode 100644 interpreter/src/ir/lower.rs diff --git a/Cargo.lock b/Cargo.lock index e672076..a6a7050 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -502,7 +502,9 @@ version = "0.1.0" dependencies = [ "color-eyre", "either", + "indexmap", "memchr", + "parser", "thiserror", "tracing", ] diff --git a/interpreter/Cargo.toml b/interpreter/Cargo.toml index 015c425..8926f0f 100644 --- a/interpreter/Cargo.toml +++ b/interpreter/Cargo.toml @@ -10,6 +10,8 @@ either.workspace = true memchr.workspace = true thiserror.workspace = true tracing.workspace = true +parser.workspace = true +indexmap = "2.14.0" [lints] workspace = true diff --git a/interpreter/src/ir.rs b/interpreter/src/ir.rs new file mode 100644 index 0000000..33f50da --- /dev/null +++ b/interpreter/src/ir.rs @@ -0,0 +1,386 @@ +// This file is part of the uutils awk package. +// +// For the full copyright and license information, please view the LICENSE +// files that was distributed with this source code. + +//! This module contains the bytecode description, designed to be compact +//! for cache efficiency and isomorphic w.r.t Cranelift IR. Also, our bytecode +//! _is_ our IR; we lower the AST into it and can execute it right away, or do +//! an optimization or JIT pass. We don't do the hack Lua 5's VM does of +//! emitting bytecode without an intermediate AST because AWK contextual +//! shenanigans; _even_ if it was possible, good luck maintaining that. + +#![allow(dead_code)] + +mod lower; + +use std::{ + fmt::{Debug, Display}, + ops::Deref, +}; + +pub use lower::test_interpreter; + +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +struct NonLocal(u16); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(transparent)] +struct Reg(u16); + +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +struct Label(u16); + +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +struct ArgCount(u16); + +#[repr(u8, align(1))] +#[derive(Clone, Copy, Debug)] +enum OpCode { + // Unary operations + Record, + Negation, + ToInt, + Negative, + Concat, + + // Binary operations + Eq, + NEq, + Gt, + Lt, + LtE, + GtE, + And, + Or, + Matches, + MatchesNot, + Add, + Subtract, + Multiply, + Divide, + Raise, + Modulo, + + // Intrinsic operations + Load, + LoadConst, + Copy, + Store, + IntrinsicCall, + UserCall, + IndirectCall, + Jump, + Return, + Branch, + BrIf, +} + +const _: () = const { assert!(size_of::() <= 8) }; + +#[derive(Clone, Copy)] +#[repr(C, align(8))] +struct Instruction { + opcode: OpCode, + // hint: Hint, + args: Arguments, +} + +#[derive(Clone, Copy)] +#[repr(C, align(2))] +union Arguments { + unary_local: (Reg, Reg), + binary_local: (Reg, Reg, Reg), + load_store: (Reg, NonLocal), + jump: Label, + ret: Reg, + branch: (Reg, Label, Label), + br_if: (Reg, Label), + call: (Reg, NonLocal, ArgCount), + ind_call: (Reg, Reg, ArgCount), +} + +impl Instruction { + fn unary(opcode: impl Into, dest: Reg, src: &impl Deref) -> Self { + let opcode = opcode.into(); + debug_assert!(opcode.is_unary()); + Self { + opcode, + args: Arguments { + unary_local: (dest, **src), + }, + } + } + + fn binary( + opcode: impl Into, + dest: Reg, + lhs: &impl Deref, + rhs: &impl Deref, + ) -> Self { + let opcode = opcode.into(); + debug_assert!(opcode.is_binary()); + Self { + opcode, + args: Arguments { + binary_local: (dest, **lhs, **rhs), + }, + } + } + + fn load_store(opcode: impl Into, dest: Reg, src: NonLocal) -> Self { + let opcode = opcode.into(); + debug_assert!(opcode.is_load_store()); + Self { + opcode, + args: Arguments { + load_store: (dest, src), + }, + } + } + + fn jump(opcode: impl Into, to: Label) -> Self { + let opcode = opcode.into(); + debug_assert!(opcode.is_jump()); + Self { + opcode, + args: Arguments { jump: to }, + } + } + + fn branch(opcode: impl Into, cond: Reg, true_to: Label, false_to: Label) -> Self { + let opcode = opcode.into(); + debug_assert!(opcode.is_branch()); + Self { + opcode, + args: Arguments { + branch: (cond, true_to, false_to), + }, + } + } + + fn br_if(opcode: impl Into, cond: Reg, to: Label) -> Self { + let opcode = opcode.into(); + debug_assert!(opcode.is_branch_if()); + Self { + opcode, + args: Arguments { br_if: (cond, to) }, + } + } +} + +impl OpCode { + fn is_unary(self) -> bool { + matches!( + self, + Self::Record | Self::Negation | Self::ToInt | Self::Negative | Self::Concat + ) + } + + fn is_binary(self) -> bool { + matches!( + self, + Self::Eq + | Self::NEq + | Self::Gt + | Self::Lt + | Self::LtE + | Self::GtE + | Self::And + | Self::Or + | Self::Matches + | Self::MatchesNot + | Self::Add + | Self::Subtract + | Self::Multiply + | Self::Divide + | Self::Raise + | Self::Modulo + ) + } + + fn is_load_store(self) -> bool { + matches!(self, Self::Load | Self::Store | Self::LoadConst) + } + + fn is_jump(self) -> bool { + matches!(self, Self::Jump) + } + + fn is_branch(self) -> bool { + matches!(self, Self::Branch) + } + + fn is_branch_if(self) -> bool { + matches!(self, Self::BrIf) + } +} + +// #[derive(Clone, Copy)] +// #[repr(u8, align(1))] +// enum Hint { +// // Next instruction contains additional data; for use in pipes, etc. +// // Also allows us to move past (2^16 - 1) variables if we want to. +// } + +impl Debug for Instruction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Instruction::{:?}", self.opcode)?; + match self.opcode { + OpCode::Record + | OpCode::Negation + | OpCode::ToInt + | OpCode::Negative + | OpCode::Concat + | OpCode::Copy => { + let (dest, data) = unsafe { &self.args.unary_local }; + write!(f, "({dest:?}, {data:?})") + } + OpCode::Eq + | OpCode::NEq + | OpCode::Gt + | OpCode::Lt + | OpCode::LtE + | OpCode::GtE + | OpCode::And + | OpCode::Or + | OpCode::Matches + | OpCode::MatchesNot + | OpCode::Add + | OpCode::Subtract + | OpCode::Multiply + | OpCode::Divide + | OpCode::Raise + | OpCode::Modulo => { + let (dest, lhs, rhs) = unsafe { &self.args.binary_local }; + write!(f, "({dest:?}, {lhs:?}, {rhs:?})") + } + OpCode::Load | OpCode::LoadConst | OpCode::Store => { + let (dest, src) = unsafe { &self.args.load_store }; + write!(f, "({dest:?}, {src:?})") + } + OpCode::BrIf => { + let (cond, label) = unsafe { self.args.br_if }; + write!(f, "({cond:?}, {label:?})") + } + OpCode::Jump => { + let label = unsafe { self.args.jump }; + write!(f, "({label:?})") + } + _ => todo!(), + } + } +} + +impl Display for Instruction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.opcode { + op @ (OpCode::Record + | OpCode::Negation + | OpCode::ToInt + | OpCode::Negative + | OpCode::Concat + | OpCode::Copy) => { + let (dest, data) = unsafe { &self.args.unary_local }; + write!(f, "{dest} <- {op} {data}") + } + op @ (OpCode::Eq + | OpCode::NEq + | OpCode::Gt + | OpCode::Lt + | OpCode::LtE + | OpCode::GtE + | OpCode::And + | OpCode::Or + | OpCode::Matches + | OpCode::MatchesNot + | OpCode::Add + | OpCode::Subtract + | OpCode::Multiply + | OpCode::Divide + | OpCode::Raise + | OpCode::Modulo) => { + let (dest, lhs, rhs) = unsafe { &self.args.binary_local }; + write!(f, "{dest} <- {op} {lhs}, {rhs}") + } + op @ (OpCode::Load | OpCode::Store) => { + let (dest, src) = unsafe { &self.args.load_store }; + write!(f, "{dest} <- {op} global[{src}]") + } + op @ OpCode::LoadConst => { + let (dest, src) = unsafe { &self.args.load_store }; + write!(f, "{dest} <- {op} mem[{src}]") + } + op @ OpCode::BrIf => { + let (cond, label) = unsafe { self.args.br_if }; + write!(f, "{op} {cond}, {label}") + } + op @ OpCode::Jump => { + let label = unsafe { self.args.jump }; + write!(f, "{op} {label}") + } + _ => todo!(), + } + } +} + +impl Display for OpCode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let str = match self { + Self::Record => "rec", + Self::Negation => "not", + Self::ToInt => "int", + Self::Negative => "neg", + Self::Concat => "cat", + Self::Eq => "eq", + Self::NEq => "neq", + Self::Gt => "gt", + Self::Lt => "lt", + Self::LtE => "le", + Self::GtE => "ge", + Self::And => "and", + Self::Or => "or", + Self::Matches => "mtch", + Self::MatchesNot => "nmtch", + Self::Add => "add", + Self::Subtract => "sub", + Self::Multiply => "mul", + Self::Divide => "div", + Self::Raise => "pow", + Self::Modulo => "mod", + Self::Load => "vload", + Self::LoadConst => "cload", + Self::Store => "vstore", + Self::Copy => "cpy", + Self::IntrinsicCall => "icall", + Self::UserCall => "ucall", + Self::IndirectCall => "vcall", + Self::Jump => "jmp", + Self::Return => "ret", + Self::Branch => "br", + Self::BrIf => "brif", + }; + <_ as Display>::fmt(str, f) + } +} + +impl Display for Label { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + <_ as Display>::fmt(&self.0, f) + } +} + +impl Display for Reg { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "r{}", self.0) + } +} + +impl Display for NonLocal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} diff --git a/interpreter/src/ir/lower.rs b/interpreter/src/ir/lower.rs new file mode 100644 index 0000000..67a32a2 --- /dev/null +++ b/interpreter/src/ir/lower.rs @@ -0,0 +1,278 @@ +// This file is part of the uutils awk package. +// +// For the full copyright and license information, please view the LICENSE +// files that was distributed with this source code. + +use std::fmt::Display; +use std::hash::Hash; +use std::mem::forget; +use std::ops::Deref; + +use indexmap::IndexSet; +use parser::{Atom, BinaryOperator, Expr, ExprNode, UnaryOperator}; + +use crate::ir::{Instruction, Label, NonLocal, OpCode, Reg}; + +#[derive(Clone, Copy, Debug, PartialEq, PartialOrd)] +struct Value(f64); // TODO: use NaN-boxing. + +#[derive(Debug)] +struct Code { + bc: Bytecode, + consts: IndexSet, + free_regs: Vec, + reg_pointer: u16, +} + +#[must_use] +#[derive(Debug)] +#[repr(transparent)] +struct LinearReg(Reg); + +impl Code { + fn lower_expr(&mut self, expr: &Expr) -> LinearReg { + let dest = self.alloc_reg(); + self.lower_expr_into(expr, dest); + LinearReg(dest) + } + fn lower_expr_into(&mut self, expr: &Expr, dest: Reg) { + match expr { + Expr::Leaf(atom) => match atom { + Atom::Variable(var) => { + let src = NonLocal(crate::index_for_global(var)); + self.bc + .emit(Instruction::load_store(OpCode::Load, dest, src)); + } + &Atom::Number(n) => { + let src = self.register_const(Value(n)); + self.bc + .emit(Instruction::load_store(OpCode::LoadConst, dest, src)); + } + _ => todo!(), + }, + Expr::Node(node) => match node.as_ref() { + ExprNode::UnaryOperation(op, expr) => { + let src = self.lower_expr(expr); + self.bc.emit(Instruction::unary(*op, dest, &src)); + self.free_reg(src); + } + ExprNode::BinaryOperation(op, lhs, rhs) => { + let lhs = self.lower_expr(lhs); + let rhs = self.lower_expr(rhs); + self.bc.emit(Instruction::binary(*op, dest, &lhs, &rhs)); + self.free_reg(lhs); + self.free_reg(rhs); + } + ExprNode::Ternary(cond, true_then, false_then) => { + self.lower_expr_into(cond, dest); + + let mut state = RegsState::new(self); + let br_if = self + .bc + .emit(Instruction::br_if(OpCode::BrIf, dest, Label(0))); + + state = state.scope(self, |c| c.lower_expr_into(false_then, dest)); + + let jump = self.bc.emit(Instruction::jump(OpCode::Jump, Label(0))); + let label = Label(self.bc.len()); + + state.scope_hwm(self, |c| c.lower_expr_into(true_then, dest)); + + self.bc.nth(br_if).args.br_if.1 = label; + self.bc.nth(jump).args.jump = Label(self.bc.len()); + } + _ => todo!(), + }, + } + } + + fn alloc_reg(&mut self) -> Reg { + self.free_regs.pop().unwrap_or_else(|| { + let current = self.reg_pointer; + self.reg_pointer += 1; + Reg(current) + }) + } + + fn free_reg(&mut self, reg: LinearReg) { + self.free_regs.push(reg.into_inner()); + } + + fn register_const(&mut self, value: Value) -> NonLocal { + NonLocal(self.consts.insert_full(value).0 as u16) + } +} + +#[derive(Debug, Default, Clone)] +struct Bytecode { + code: Vec, +} + +#[derive(Clone, Debug)] +struct RegsState { + reg_pointer: u16, + n_free_regs: usize, +} + +impl Bytecode { + fn new() -> Self { + Self::default() + } + + #[inline(always)] + fn emit(&mut self, code: Instruction) -> Label { + self.code.push(code); + Label((self.code.len() - 1) as u16) + } + + fn len(&self) -> u16 { + self.code.len() as u16 + } + + fn nth(&mut self, label: Label) -> &mut Instruction { + &mut self.code[label.0 as usize] + } +} + +impl RegsState { + fn new(code: &Code) -> Self { + Self { + reg_pointer: code.reg_pointer, + n_free_regs: code.free_regs.len(), + } + } + fn scope(self, code: &mut Code, f: impl FnOnce(&mut Code)) -> Self { + f(code); + let old = code.reg_pointer; + code.reg_pointer = self.reg_pointer; + code.free_regs.truncate(self.n_free_regs); + Self { + reg_pointer: old, + n_free_regs: self.n_free_regs, + } + } + fn scope_hwm(self, code: &mut Code, f: impl FnOnce(&mut Code)) { + f(code); + code.reg_pointer = code.reg_pointer.max(self.reg_pointer); + code.free_regs.truncate(self.n_free_regs); + } +} + +pub fn test_interpreter(expr: &Expr<'_>) -> impl Display { + let mut c = Code { + bc: Bytecode::new(), + consts: IndexSet::new(), + reg_pointer: 0, + free_regs: Vec::new(), + }; + let result = c.lower_expr(expr); + dbg!(&c, &result); + forget(result); + c +} + +impl From for OpCode { + fn from(value: UnaryOperator) -> Self { + match value { + UnaryOperator::Record => Self::Record, + UnaryOperator::Negation => Self::Negation, + UnaryOperator::ToInt => Self::ToInt, + UnaryOperator::Negative => Self::Negative, + } + } +} + +impl From for OpCode { + fn from(value: BinaryOperator) -> Self { + match value { + BinaryOperator::Concat => Self::Concat, + BinaryOperator::Eq => Self::Eq, + BinaryOperator::NEq => Self::NEq, + BinaryOperator::Gt => Self::Gt, + BinaryOperator::Lt => Self::Lt, + BinaryOperator::LtE => Self::LtE, + BinaryOperator::GtE => Self::GtE, + BinaryOperator::And => Self::And, + BinaryOperator::Or => Self::Or, + BinaryOperator::Matches => Self::Matches, + BinaryOperator::MatchesNot => Self::MatchesNot, + BinaryOperator::Add => Self::Add, + BinaryOperator::Subtract => Self::Subtract, + BinaryOperator::Multiply => Self::Multiply, + BinaryOperator::Divide => Self::Divide, + BinaryOperator::Raise => Self::Raise, + BinaryOperator::Modulo => Self::Modulo, + } + } +} + +impl Hash for Value { + fn hash(&self, state: &mut H) { + state.write(&self.0.to_ne_bytes()); + } +} + +impl Eq for Value {} + +// trait Fold { +// type Args; +// fn fold(&self, args: Self::Args) -> T; +// } + +impl Display for Bytecode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let n = self.code.len() / 10 + 1; + for (i, e) in self.code.iter().enumerate() { + write!(f, "{i:n$}: {e}")?; + if i + 1 < self.code.len() as _ { + writeln!(f)?; + } + } + Ok(()) + } +} + +impl Display for Code { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "Bytecode:\n{}\n", self.bc)?; + writeln!(f, "Consts:")?; + for (i, e) in self.consts.iter().enumerate() { + write!(f, "mem[{i}] = {}", e.0)?; + if i + 1 < self.consts.len() as _ { + writeln!(f)?; + } + } + Ok(()) + } +} + +impl LinearReg { + fn into_inner(self) -> Reg { + let inner = self.0; + forget(self); + inner + } +} + +impl Deref for LinearReg { + type Target = Reg; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +// impl Deref for Reg { +// type Target = Self; + +// fn deref(&self) -> &Self::Target { +// self +// } +// } + +#[cfg(debug_assertions)] +impl Drop for LinearReg { + fn drop(&mut self) { + debug_assert!(false, "Leaked register {}!", self.0); + } +} diff --git a/interpreter/src/lib.rs b/interpreter/src/lib.rs index 7c9f462..6c11358 100644 --- a/interpreter/src/lib.rs +++ b/interpreter/src/lib.rs @@ -3,14 +3,19 @@ // For the full copyright and license information, please view the LICENSE // files that was distributed with this source code. +mod ir; + use color_eyre::eyre::Result; use either::Either; +pub use ir::test_interpreter; +use parser::Variable; + pub enum BuiltinCommand {} pub enum BuiltinVar {} pub type Command<'a> = Either; -pub type Variable<'a> = Either; +// pub type Variable<'a> = Either; #[derive(Debug)] pub struct Interpreter; @@ -23,6 +28,10 @@ impl Interpreter { pub fn eval_expression(&mut self) {} } +fn index_for_global(_var: &Variable) -> u16 { + 1 // TODO +} + #[derive(Debug, thiserror::Error)] #[allow(dead_code)] enum InterpreterError {} diff --git a/parser/src/ast.rs b/parser/src/ast.rs index bb4f6bd..ce11db7 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -149,7 +149,6 @@ pub enum BinaryPlaceOperator { PowAssign, ModAssign, } - #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum ArrayOperator { Index, diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 4fd5b18..be6f613 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -21,13 +21,9 @@ use either::Either::{Left, Right}; use hashbrown::HashMap; use lexer::{LexingError, Span, Token}; -pub use crate::ast::Ast; +pub use crate::ast::*; pub use crate::lex::Lexer; use crate::{ - ast::{ - ArrayOperator, Atom, Body, Command, Expr, ExprNode, Function, Identifier, Pattern, - Redirection, Rule, RulePattern, SimpleStatement, SpecialPattern, Statement, Variable, - }, diagnostics::{ParsingError, report_error}, lex::TokenExt, pratt::Pratt, diff --git a/src/main.rs b/src/main.rs index 437126f..f5b4c94 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,7 +14,8 @@ use std::io::{self, Write}; use bumpalo::Bump; use clap::Parser as _; use color_eyre::Result; -use parser::Parser; +use interpreter::test_interpreter; +use parser::{Body, Parser, Rule}; use crate::{ cli::Args, @@ -53,6 +54,20 @@ fn uu_main() -> Result<()> { } dbg!(arena.chunk_capacity()); + if let Some(Rule { + actions: Some(Body(a)), + pattern: _, + }) = ast.rules.first() + && let Some(parser::Statement::Simple(parser::SimpleStatement::Expression(expr))) = + a.first() + { + let x = test_interpreter(expr); + if let Err(e) = writeln!(io::stdout(), "---\n{x}") + && e.kind() != io::ErrorKind::BrokenPipe + { + exit_err(Some(format!("awk: error writing to standard output: {e}"))); + } + } // for token in lex { // let Ok(x) = token else { // return token.map(drop).map_err(color_eyre::Report::from);