commit 1609ff1b1cb6c74931bb7cb18df2b480ec13a6f7 Author: Guillem L. Jara <4lon3ly0@tutanota.com> Date: Wed Apr 29 21:59:04 2026 +0200 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..905c026 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,545 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "allocator-api2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880" + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +dependencies = [ + "allocator-api2 0.2.21", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "color-eyre" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5920befb47832a6d61ee3a3a846565cfa39b331331e68a3b1d1116630f2f26d" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "allocator-api2 0.2.21", + "equivalent", + "foldhash", +] + +[[package]] +name = "indenter" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" + +[[package]] +name = "interpreter" +version = "0.1.0" +dependencies = [ + "color-eyre", + "either", + "memchr", + "thiserror", + "tracing", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lexer" +version = "0.1.0" +dependencies = [ + "logos", + "memchr", + "thiserror", + "tracing", +] + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "logos" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2c55a318a87600ea870ff8c2012148b44bf18b74fad48d0f835c38c7d07c5f" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58b3ffaa284e1350d017a57d04ada118c4583cf260c8fb01e0fe28a2e9cf8970" +dependencies = [ + "fnv", + "proc-macro2", + "quote", + "regex-automata", + "regex-syntax", + "syn", +] + +[[package]] +name = "logos-derive" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d3a9855747c17eaf4383823f135220716ab49bea5fbea7dd42cc9a92f8aa31" +dependencies = [ + "logos-codegen", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "owo-colors" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" + +[[package]] +name = "parser" +version = "0.1.0" +dependencies = [ + "allocator-api2 0.4.0", + "bumpalo", + "either", + "hashbrown", + "lexer", + "memchr", + "smallvec", + "thiserror", + "tracing", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "uu_awk" +version = "0.1.0" +dependencies = [ + "bumpalo", + "color-eyre", + "either", + "interpreter", + "memchr", + "parser", + "rustix", + "thiserror", + "tracing", + "tracing-error", + "tracing-subscriber", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..0e02d9d --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,129 @@ +[package] +name = "uu_awk" +version.workspace = true +edition.workspace = true +license.workspace = true + +[[bin]] +name = "awk" +path = "src/main.rs" + +[workspace.package] +version = "0.1.0" +license = "MIT OR Apache-2.0" +edition = "2024" + +[workspace] +resolver = "3" +members = ["interpreter","lexer", "parser"] + +[workspace.dependencies] +lexer.path = "./lexer" +parser.path = "./parser" +interpreter.path = "./interpreter" +color-eyre = "0.6.5" +either = "1.15.0" +memchr = "2.8.0" +thiserror = "2.0.18" +tracing = "0.1.44" +smallvec = { version = "1.15.1", features = ["union", "const_generics", "const_new"] } +bumpalo = { version = "3.20.2", features = ["boxed", "collections", "std", "allocator-api2"] } +hashbrown = "0.17.0" +allocator-api2 = "0.4.0" + +[dependencies] +# lexer.workspace = true # FIXME(parser complete): bypass through parser +interpreter.workspace = true +parser.workspace = true +color-eyre.workspace = true +either.workspace = true +memchr.workspace = true +thiserror.workspace = true +tracing.workspace = true +bumpalo.workspace = true +rustix = { version = "1.1.4", default-features = false, features = ["process"]} +tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } +tracing-error = "0.2.1" + +[profile.release] +lto = true +panic = "abort" +codegen-units = 1 +debug = true + +# A release-like profile that is as small as possible. +[profile.release-small] +inherits = "release" +opt-level = "z" +strip = true + +[profile.dev.package.backtrace] +opt-level = 3 + +# A release-like profile with debug info for profiling. +# See https://github.com/mstange/samply . +[profile.profiling] +inherits = "release" +panic = "unwind" +debug = true + +[lints] +workspace = true + +# This is the linting configuration for all crates. +# In order to use these, all crates have `[lints] workspace = true` section. +[workspace.lints.rust] +# Allow "fuzzing" as a "cfg" condition name and "cygwin" as a value for "target_os" +# https://doc.rust-lang.org/nightly/rustc/check-cfg/cargo-specifics.html +unexpected_cfgs = { level = "warn", check-cfg = [ + 'cfg(fuzzing)', + 'cfg(target_os, values("cygwin"))', + 'cfg(wasi_runner)', +] } +unused_qualifications = "warn" + +[workspace.lints.clippy] +collapsible_if = { level = "allow", priority = 127 } # remove me +# The counts were generated with this command: +# cargo clippy --all-targets --workspace --message-format=json --quiet \ +# | jq -r '.message.code.code | select(. != null and startswith("clippy::"))' \ +# | sort | uniq -c | sort -h -r +# +all = { level = "warn", priority = -1 } +cargo = { level = "warn", priority = -1 } +pedantic = { level = "warn", priority = -1 } +use_self = "warn" # nursery lint +cargo_common_metadata = "allow" # 3240 +multiple_crate_versions = "allow" # 2882 +missing_errors_doc = "allow" # 1572 +missing_panics_doc = "allow" # 946 +must_use_candidate = "allow" # 322 +match_same_arms = "allow" # 204 +cast_possible_truncation = "allow" # 122 +too_many_lines = "allow" # 101 +cast_possible_wrap = "allow" # 78 +cast_sign_loss = "allow" # 70 +struct_excessive_bools = "allow" # 68 +cast_precision_loss = "allow" # 52 +cast_lossless = "allow" # 35 +ignored_unit_patterns = "allow" # 21 +similar_names = "allow" # 20 +needless_pass_by_value = "allow" # 16 +float_cmp = "allow" # 12 +items_after_statements = "allow" # 11 +return_self_not_must_use = "allow" # 8 +inline_always = "allow" # 6 +fn_params_excessive_bools = "allow" # 6 +used_underscore_items = "allow" # 2 +should_panic_without_expect = "allow" # 2 + +doc_markdown = "allow" +unused_self = "allow" +enum_glob_use = "allow" +unnested_or_patterns = "allow" +implicit_hasher = "allow" +doc_link_with_quotes = "allow" +format_push_string = "allow" +flat_map_option = "allow" +from_iter_instead_of_collect = "allow" +large_types_passed_by_value = "allow" diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 0000000..e6a70d5 --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,179 @@ +Copyright (c) 2026- Guillem Larrosa Jara +Copyright (c) 2026- The uutils AWK Contributors + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..2b75c19 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2026- Guillem Larrosa Jara +Copyright (c) 2026- The uutils AWK Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ab91b93 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# uutils AWK + +This is a human, WIP, and clean implementation of an AWK interpreter, written in Rust and compatible with GNU's AWK (`gawk`) bug-for-bug. Expected to be production-ready before Ubuntu 26.10. Made with love. + +## State of the Repo + +The lexer is pretty much done; however, it should track the span of the tokens so we can produce contextual error messages in the parser, and it is also lacking a POSIX-compatibility mode (trivial). The parser is mostly done, although it probably has some rough edges, and the preprocessor is TBD; also, the Pratt routine is a bit spaghetti and should be refactored (trivial). The interpreter is also TBD; work on it will be started once we get good testing on the parser. Expect this to be a fast-paced repo. + +## Contributing + +See [this](https://github.com/uutils/coreutils/blob/main/CONTRIBUTING.md). + +## License + +This is licensed under either the MIT License or the Apache License v2.0. See the `LICENSE-MIT` and `LICENSE-APACHE` files for details. diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..6eaced6 --- /dev/null +++ b/deny.toml @@ -0,0 +1,240 @@ +# This template contains all of the possible sections and their default values + +# Note that all fields that take a lint level have these possible values: +# * deny - An error will be produced and the check will fail +# * warn - A warning will be produced, but the check will not fail +# * allow - No warning or error will be produced, though in some cases a note +# will be + +# The values provided in this template are the default values that will be used +# when any section or field is not specified in your own configuration + +# Root options + +# The graph table configures how the dependency graph is constructed and thus +# which crates the checks are performed against +[graph] +# If 1 or more target triples (and optionally, target_features) are specified, +# only the specified targets will be checked when running `cargo deny check`. +# This means, if a particular package is only ever used as a target specific +# dependency, such as, for example, the `nix` crate only being used via the +# `target_family = "unix"` configuration, that only having windows targets in +# this list would mean the nix crate, as well as any of its exclusive +# dependencies not shared by any other crates, would be ignored, as the target +# list here is effectively saying which targets you are building for. +targets = [ + # The triple can be any string, but only the target triples built in to + # rustc (as of 1.40) can be checked against actual config expressions + #"x86_64-unknown-linux-musl", + # You can also specify which target_features you promise are enabled for a + # particular target. target_features are currently not validated against + # the actual valid features supported by the target architecture. + #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, +] +# When creating the dependency graph used as the source of truth when checks are +# executed, this field can be used to prune crates from the graph, removing them +# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate +# is pruned from the graph, all of its dependencies will also be pruned unless +# they are connected to another crate in the graph that hasn't been pruned, +# so it should be used with care. The identifiers are [Package ID Specifications] +# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html) +#exclude = [] +# If true, metadata will be collected with `--all-features`. Note that this can't +# be toggled off if true, if you want to conditionally enable `--all-features` it +# is recommended to pass `--all-features` on the cmd line instead +all-features = false +# If true, metadata will be collected with `--no-default-features`. The same +# caveat with `all-features` applies +no-default-features = false +# If set, these feature will be enabled when collecting metadata. If `--features` +# is specified on the cmd line they will take precedence over this option. +#features = [] + +# The output table provides options for how/if diagnostics are outputted +[output] +# When outputting inclusion graphs in diagnostics that include features, this +# option can be used to specify the depth at which feature edges will be added. +# This option is included since the graphs can be quite large and the addition +# of features from the crate(s) to all of the graph roots can be far too verbose. +# This option can be overridden via `--feature-depth` on the cmd line +feature-depth = 1 + +# This section is considered when running `cargo deny check advisories` +# More documentation for the advisories section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html +[advisories] +# The path where the advisory databases are cloned/fetched into +#db-path = "$CARGO_HOME/advisory-dbs" +# The url(s) of the advisory databases to use +#db-urls = ["https://github.com/rustsec/advisory-db"] +# A list of advisory IDs to ignore. Note that ignored advisories will still +# output a note when they are encountered. +ignore = [ + #"RUSTSEC-0000-0000", + #{ id = "RUSTSEC-0000-0000", reason = "you can specify a reason the advisory is ignored" }, + #"a-crate-that-is-yanked@0.1.1", # you can also ignore yanked crate versions if you wish + #{ crate = "a-crate-that-is-yanked@0.1.1", reason = "you can specify why you are ignoring the yanked crate" }, +] +# If this is true, then cargo deny will use the git executable to fetch advisory database. +# If this is false, then it uses a built-in git library. +# Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support. +# See Git Authentication for more information about setting up git authentication. +#git-fetch-with-cli = true + +# This section is considered when running `cargo deny check licenses` +# More documentation for the licenses section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html +[licenses] +# List of explicitly allowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +allow = [ + "MIT", + "Apache-2.0", + # "Apache-2.0 WITH LLVM-exception", + "Unicode-3.0" +] +# The confidence threshold for detecting a license from license text. +# The higher the value, the more closely the license text must be to the +# canonical license text of a valid SPDX license file. +# [possible values: any between 0.0 and 1.0]. +confidence-threshold = 0.8 +# Allow 1 or more licenses on a per-crate basis, so that particular licenses +# aren't accepted for every possible crate as with the normal allow list +exceptions = [ + # Each entry is the crate and version constraint, and its specific allow + # list + #{ allow = ["Zlib"], crate = "adler32" }, +] + +# Some crates don't have (easily) machine readable licensing information, +# adding a clarification entry for it allows you to manually specify the +# licensing information +#[[licenses.clarify]] +# The package spec the clarification applies to +#crate = "ring" +# The SPDX expression for the license requirements of the crate +#expression = "MIT AND ISC AND OpenSSL" +# One or more files in the crate's source used as the "source of truth" for +# the license expression. If the contents match, the clarification will be used +# when running the license check, otherwise the clarification will be ignored +# and the crate will be checked normally, which may produce warnings or errors +# depending on the rest of your configuration +#license-files = [ +# Each entry is a crate relative path, and the (opaque) hash of its contents +#{ path = "LICENSE", hash = 0xbd0eed23 } +#] + +[licenses.private] +# If true, ignores workspace crates that aren't published, or are only +# published to private registries. +# To see how to mark a crate as unpublished (to the official registry), +# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field. +ignore = false +# One or more private registries that you might publish crates to, if a crate +# is only published to private registries, and ignore is true, the crate will +# not have its license(s) checked +registries = [ + #"https://sekretz.com/registry +] + +# This section is considered when running `cargo deny check bans`. +# More documentation about the 'bans' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html +[bans] +# Lint level for when multiple versions of the same crate are detected +multiple-versions = "warn" +# Lint level for when a crate version requirement is `*` +wildcards = "allow" +# The graph highlighting used when creating dotgraphs for crates +# with multiple versions +# * lowest-version - The path to the lowest versioned duplicate is highlighted +# * simplest-path - The path to the version with the fewest edges is highlighted +# * all - Both lowest-version and simplest-path are used +highlight = "all" +# The default lint level for `default` features for crates that are members of +# the workspace that is being checked. This can be overridden by allowing/denying +# `default` on a crate-by-crate basis if desired. +workspace-default-features = "allow" +# The default lint level for `default` features for external crates that are not +# members of the workspace. This can be overridden by allowing/denying `default` +# on a crate-by-crate basis if desired. +external-default-features = "allow" +# List of crates that are allowed. Use with care! +allow = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is allowed" }, +] +# If true, workspace members are automatically allowed even when using deny-by-default +# This is useful for organizations that want to deny all external dependencies by default +# but allow their own workspace crates without having to explicitly list them +allow-workspace = false +# List of crates to deny +deny = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is banned" }, + # Wrapper crates can optionally be specified to allow the crate when it + # is a direct dependency of the otherwise banned crate + #{ crate = "ansi_term@0.11.0", wrappers = ["this-crate-directly-depends-on-ansi_term"] }, +] + +# List of features to allow/deny +# Each entry the name of a crate and a version range. If version is +# not specified, all versions will be matched. +#[[bans.features]] +#crate = "reqwest" +# Features to not allow +#deny = ["json"] +# Features to allow +#allow = [ +# "rustls", +# "__rustls", +# "__tls", +# "hyper-rustls", +# "rustls", +# "rustls-pemfile", +# "rustls-tls-webpki-roots", +# "tokio-rustls", +# "webpki-roots", +#] +# If true, the allowed features must exactly match the enabled feature set. If +# this is set there is no point setting `deny` +#exact = true + +# Certain crates/versions that will be skipped when doing duplicate detection. +skip = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason why it can't be updated/removed" }, +] +# Similarly to `skip` allows you to skip certain crates during duplicate +# detection. Unlike skip, it also includes the entire tree of transitive +# dependencies starting at the specified crate, up to a certain depth, which is +# by default infinite. +skip-tree = [ + #"ansi_term@0.11.0", # will be skipped along with _all_ of its direct and transitive dependencies + #{ crate = "ansi_term@0.11.0", depth = 20 }, +] + +# This section is considered when running `cargo deny check sources`. +# More documentation about the 'sources' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html +[sources] +# Lint level for what to happen when a crate from a crate registry that is not +# in the allow list is encountered +unknown-registry = "warn" +# Lint level for what to happen when a crate from a git repository that is not +# in the allow list is encountered +unknown-git = "warn" +# List of URLs for allowed crate registries. Defaults to the crates.io index +# if not specified. If it is specified but empty, no registries are allowed. +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +# List of URLs for allowed Git repositories +allow-git = [] + +[sources.allow-org] +# github.com organizations to allow git sources for +github = [] +# gitlab.com organizations to allow git sources for +gitlab = [] +# bitbucket.org organizations to allow git sources for +bitbucket = [] diff --git a/interpreter/Cargo.toml b/interpreter/Cargo.toml new file mode 100644 index 0000000..015c425 --- /dev/null +++ b/interpreter/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "interpreter" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +color-eyre.workspace = true +either.workspace = true +memchr.workspace = true +thiserror.workspace = true +tracing.workspace = true + +[lints] +workspace = true diff --git a/interpreter/src/lib.rs b/interpreter/src/lib.rs new file mode 100644 index 0000000..d0c8036 --- /dev/null +++ b/interpreter/src/lib.rs @@ -0,0 +1,22 @@ +use color_eyre::eyre::Result; +use either::Either; + +pub enum BuiltinCommand {} +pub enum BuiltinVar {} + +pub type Command<'a> = Either; +pub type Variable<'a> = Either; + +#[derive(Debug)] +pub struct Interpreter; + +impl Interpreter { + #[tracing::instrument] + pub fn run(self) -> Result> { + todo!() + } + pub fn eval_expression(&mut self) {} +} + +#[derive(Debug, thiserror::Error)] +enum InterpreterError {} diff --git a/lexer/Cargo.toml b/lexer/Cargo.toml new file mode 100644 index 0000000..b564b81 --- /dev/null +++ b/lexer/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "lexer" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +logos = "0.16.1" +# bytecount = "0.6.9" +memchr.workspace = true +thiserror.workspace = true +tracing.workspace = true + +[lints] +workspace = true diff --git a/lexer/src/lib.rs b/lexer/src/lib.rs new file mode 100644 index 0000000..862ce2f --- /dev/null +++ b/lexer/src/lib.rs @@ -0,0 +1,412 @@ +#[cfg(test)] +mod tests; + +use core::str; +use std::{borrow::Cow, fmt::Debug, slice::SliceIndex}; + +use logos::Skip; +pub use logos::{Logos, Span}; +use memchr::{memchr, memchr3}; +use thiserror::Error; + +pub type Lexer<'a> = logos::Lexer<'a, Token<'a>>; +pub type Result = std::result::Result; + +// TODO: check wnat GNU does about potentially reserved `@ident`; add error branch if so. +#[derive(Logos, Debug, PartialEq)] +#[logos(utf8 = false)] +#[logos(skip("[ \t]+"))] +#[logos(skip(r"(\\\n)+"))] +#[logos(skip("#", skip_line))] +#[logos(extras = Context)] +#[logos(subpattern identifier = r"[a-zA-Z_][a-zA-Z0-9_]*")] +#[logos(error(LexingError, callback = |lex| LexingError::from(lex)))] +pub enum Token<'a> { + #[regex(r"[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?", parse_float)] + #[regex(r"\.[0-9]+([eE][+-]?[0-9]+)?", parse_float)] + Number(f64), + #[token("\"", parse_string)] + String(Slice<'a>), + #[token("BEGIN", accept_expression)] + BeginPattern, + #[token("END", accept_expression)] + EndPattern, + #[token("BEGINFILE", accept_expression)] + BeginFilePattern, + #[token("ENDFILE", accept_expression)] + EndFilePattern, + #[token("@load \"", parse_directive)] + LoadDirective(Slice<'a>), + #[token("@include \"", parse_directive)] + IncludeDirective(Slice<'a>), + #[token("@nsinclude \"", parse_directive)] + NsIncludeDirective(Slice<'a>), + #[regex("@namespace \"(?&identifier)\"", |lex| parse_ident(lex, 12..lex.slice().len() - 1))] + NamespaceDirective(&'a str), + #[token("@concurrent", accept_expression)] + ConcurrentDirective, + #[token("if", accept_expression)] + If, + #[regex(r"else\n*", accept_expression)] + Else, + #[token("switch", accept_expression)] + Switch, + #[token("case", accept_expression)] + Case, + #[token("default", accept_expression)] + Default, + #[regex(r"do\n*", accept_expression)] + Do, + #[token("while", accept_expression)] + While, + #[token("for", accept_expression)] + For, + #[token("in", accept_expression)] + In, + #[token("print", accept_expression)] + Print, + #[token("printf", accept_expression)] + Printf, + #[token("getline", accept_expression)] + Getline, + #[token("next", accept_expression)] + Next, + #[token("nextfile", accept_expression)] + NextFile, + #[token("exit", accept_expression)] + Exit, + #[token("break", accept_expression)] + Break, + #[token("continue", accept_expression)] + Continue, + #[token("return", accept_expression)] + Return, + #[token("delete", accept_expression)] + Delete, + #[token("function", accept_expression)] + Function, + #[token("NR", accept_expression)] + NrVariable, + #[token("NF", accept_expression)] + NfVariable, + #[token("FS", accept_expression)] + FsVariable, + #[token("RS", accept_expression)] + RsVariable, + #[token("OFS", accept_expression)] + OfsVariable, + #[token("ORS", accept_expression)] + OrsVariable, + #[token("FILENAME", accept_expression)] + FilenameVariable, + #[token("ARGC", accept_expression)] + ArgcVariable, + #[token("ARGV", accept_expression)] + ArgvVariable, + #[token("SUBSEP", accept_expression)] + SubsepVariable, + #[token("FNR", accept_expression)] + FnrVariable, + #[token("ARGIND")] + ArgindVariable, + #[token("OFMT", accept_expression)] + OfmtVariable, + #[token("RSTART", accept_expression)] + RstartVariable, + #[token("RLENGTH", accept_expression)] + RlengthVariable, + #[token("ENVIRON", accept_expression)] + EnvironVariable, + #[regex("(?&identifier)", Identifier::without_namespace)] + #[regex(r"(?&identifier)::(?&identifier)", Identifier::with_namespace)] + Identifier(Identifier<'a>), + #[regex(r"(?&identifier)\(", Identifier::without_namespace)] + #[regex(r"(?&identifier)::(?&identifier)\(", Identifier::with_namespace)] + FunctionCall(Identifier<'a>), + #[token("+", accept_expression)] + Plus, + #[token("-", accept_expression)] + Minus, + #[token("*", accept_expression)] + Star, + #[token("/", parse_regex_or_slash)] + Slash, + /// Not generated by Logos directly. + Regex(Slice<'a>), + #[token("%", accept_expression)] + Percent, + #[token("^", accept_expression)] + Circumflex, + #[token("++", accept_expression)] + Increment, + #[token("--", accept_expression)] + Decrement, + #[token("=", accept_expression)] + Assignment, + #[token("+=", accept_expression)] + PlusAssign, + #[token("-=", accept_expression)] + MinusAssign, + #[token("*=", accept_expression)] + StarAssign, + #[token("/=", accept_expression)] + SlashAssign, + #[token("%=", accept_expression)] + PercentAssign, + #[token("^=", accept_expression)] + CaretAssign, + #[token("==", accept_expression)] + EqualTo, + #[token("!=", accept_expression)] + NotEqualTo, + #[token("<", accept_expression)] + LesserThan, + #[token("<=", accept_expression)] + LesserOrEqualThan, + #[token(">", accept_expression)] + GreaterThan, + #[token(">=", accept_expression)] + GreaterOrEqualThan, + #[regex(r"&&\n*", accept_expression)] + BooleanAnd, + #[regex(r"\|\|\n*", accept_expression)] + BooleanOr, + #[token("!", accept_expression)] + Negation, + #[token("~", accept_expression)] + Matching, + #[token("!~", accept_expression)] + NotMatching, + #[token("|", accept_expression)] + Pipe, + #[token("|&", accept_expression)] + DoublePipe, + #[token(">>", accept_expression)] + AppendPipe, + #[regex(r"\?\n*", accept_expression)] + QuestionMark, + #[regex(r":\n*", accept_expression)] + Colon, + #[regex(r"\{\n*", accept_expression)] + OpenBrace, + #[token("}", accept_expression)] + ClosedBrace, + #[token("(", accept_expression)] + OpenParent, + #[token(")", accept_operator)] + ClosedParent, + #[token("[", accept_expression)] + OpenBracket, + #[token("]", accept_operator)] + ClosedBracket, + #[regex(r",\n*", accept_expression)] + Comma, + #[token("$", accept_expression)] + Record, + #[regex(r"\n+", accept_expression)] + Newline, + #[regex(";+", accept_expression)] + Semicolon, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub enum Context { + #[default] + AcceptExpression, + AcceptOperator, +} + +#[derive(Debug, Default, Error, Clone, PartialEq, Eq)] +pub enum LexingError { + #[default] + #[error("Unknown error")] + Unknown, + #[error("Unexpected token at {:?}: `{}`", .0, .1)] + Unexpected(Span, String), + #[error("Unterminated string at {:?}", .0)] + UnterminatedString(Span), + #[error("Unterminated regex at {:?}", .0)] + UnterminatedRegex(Span), + #[error("Unexpected End of File!")] + UnexpectedEof, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Identifier<'a> { + pub namespace: Option<&'a str>, + pub literal: &'a str, +} + +impl From<&mut Lexer<'_>> for LexingError { + fn from(lex: &mut Lexer<'_>) -> Self { + Self::Unexpected(lex.span(), String::from_utf8_lossy(lex.slice()).to_string()) + } +} + +fn skip_line(lex: &mut Lexer<'_>) -> Skip { + lex.bump(memchr(b'\n', lex.remainder()).unwrap_or(lex.remainder().len())); + logos::skip(lex) +} + +fn parse_string<'a>(lex: &mut logos::Lexer<'a, Token<'a>>) -> Result> { + parse_content::(lex) +} + +fn parse_regex_or_slash<'a>(lex: &mut logos::Lexer<'a, Token<'a>>) -> Result> { + match lex.extras { + Context::AcceptExpression => { + accept_operator(lex); + parse_content::(lex).map(Token::Regex) + } + Context::AcceptOperator => { + accept_expression(lex); + Ok(Token::Slash) + } + } +} + +fn parse_directive<'a>(lex: &mut Lexer<'a>) -> Result> { + parse_content::(lex) +} + +fn parse_content<'a, const MINIMAL: bool, const REGEX: bool, const DELIMITER: char>( + lex: &mut Lexer<'a>, +) -> Result> { + let rest = lex.remainder(); + let mut start = 0; + let mut out: Cow<'a, [u8]> = Cow::Borrowed(&[]); + + while let Some(rel_i) = memchr3(b'\n', b'\\', DELIMITER as u8, &rest[start..]) { + let i = start + rel_i; + + match rest[i] { + d if d == DELIMITER as u8 => { + // push remaining segment + lex.bump(i + 1); + if start == 0 { + out = Cow::Borrowed(&rest[..i]); + } else { + out.to_mut().extend_from_slice(&rest[start..i]); + } + return Ok(Slice(out)); + } + b'\\' => { + out.to_mut().extend_from_slice(&rest[start..i]); + let consumed = parse_escape::(&rest[i..], out.to_mut())?; + start = i + consumed; + } + _ => break, + } + } + if REGEX { + Err(LexingError::UnterminatedRegex(lex.span())) + } else { + Err(LexingError::UnterminatedString(lex.span())) + } +} + +fn parse_escape( + slice: &[u8], + out: &mut Vec, +) -> Result { + let mut count = 2; + let is_oct = |x: char| ('0'..'8').contains(&x); + let is_slice_oct = |i| slice.get(i).map(|&x| x as char).is_some_and(is_oct); + let Some(to_escape) = slice.get(1).map(|x| *x as char) else { + return Err(LexingError::UnexpectedEof); + }; + + if to_escape == '\n' { + return Ok(count); + } + + // On minimal, only drop backslash for '"' and '\n' + let escaped = if MINIMAL && !matches!(to_escape, '"' | '\n') { + out.push(b'\\'); + to_escape + } else if MINIMAL { + to_escape + } else { + match to_escape { + c @ ('\\' | '"') if !REGEX => c, + c @ ('[' | ']' | '{' | '}' | '(' | ')' | '*' | '+' | '^' | '$' | '.' | '?') + if REGEX => + { + out.push(b'\\'); + c + } + 'a' => 7 as char, + 'b' => 8 as char, + 'f' => 12 as char, + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'v' => 11 as char, + n if is_oct(n) => { + count += is_slice_oct(2) as usize + (is_slice_oct(2) && is_slice_oct(3)) as usize; + slice[1..] + .iter() + .take(count - 1) + .fold(0, |acc, digit| acc * 8 + digit - b'0') as char + } + 'x' => todo!(), + 'u' => todo!(), + // Unspecified by POSIX; we ditto GNU. + c => c, // TODO: Output warning + } + }; + out.push(escaped as u8); + Ok(count) +} + +fn parse_ident<'a>(lex: &mut Lexer<'a>, index: impl SliceIndex<[u8], Output = [u8]>) -> &'a str { + accept_operator(lex); + // SAFETY: The regex matching ensures it is ASCII. + unsafe { str::from_utf8_unchecked(lex.slice().get_unchecked(index)) } +} + +fn parse_float(lex: &mut Lexer<'_>) -> f64 { + parse_ident(lex, ..).parse().unwrap_or(0.) +} + +impl<'a> Identifier<'a> { + fn without_namespace(lex: &mut Lexer<'a>) -> Self { + Self { + namespace: None, + literal: parse_ident(lex, ..), + } + } + + fn with_namespace(lex: &mut Lexer<'a>) -> Self { + // SAFETY: The regex matching ensures it is present and well-formed. + let separator = unsafe { memchr(b':', lex.slice()).unwrap_unchecked() }; + Self { + namespace: Some(parse_ident(lex, ..separator)), + literal: parse_ident(lex, separator + 2..), + } + } +} + +fn accept_expression(lex: &mut Lexer<'_>) { + lex.extras = Context::AcceptExpression; +} + +fn accept_operator(lex: &mut Lexer<'_>) { + lex.extras = Context::AcceptOperator; +} + +#[repr(transparent)] +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone)] +pub struct Slice<'a>(Cow<'a, [u8]>); + +impl Debug for Slice<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "\"{}\"", String::from_utf8_lossy(&self.0).as_ref()) + } +} + +impl AsRef<[u8]> for Slice<'_> { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} diff --git a/parser/Cargo.toml b/parser/Cargo.toml new file mode 100644 index 0000000..dbd6d4c --- /dev/null +++ b/parser/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "parser" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +lexer.workspace = true +either.workspace = true +memchr.workspace = true +thiserror.workspace = true +tracing.workspace = true +smallvec.workspace = true +bumpalo.workspace = true +hashbrown.workspace = true +allocator-api2.workspace = true + +[lints] +workspace = true diff --git a/parser/src/ast.rs b/parser/src/ast.rs new file mode 100644 index 0000000..6fecf76 --- /dev/null +++ b/parser/src/ast.rs @@ -0,0 +1,582 @@ +use std::fmt::Debug; + +use bumpalo::{Bump, collections::Vec}; +use either::Either; +use hashbrown::{DefaultHashBuilder, HashMap}; +use lexer::{Slice, Token}; + +use crate::{ParsingError, Result, lex::TokenExt}; + +#[derive(Debug, Clone)] +pub struct Ast<'a> { + pub loads: Vec<'a, Slice<'a>>, + pub begin: Vec<'a, Body<'a>>, + pub end: Vec<'a, Body<'a>>, + pub begin_file: Vec<'a, Body<'a>>, + pub end_file: Vec<'a, Body<'a>>, + pub rules: Vec<'a, Rule<'a>>, + pub concurrent: Vec<'a, Rule<'a>>, + pub functions: HashMap, Function<'a>, DefaultHashBuilder, &'a Bump>, +} + +#[derive(Debug, Clone)] +pub struct Rule<'a> { + pub pattern: Option>, + pub actions: Option>, +} + +#[derive(Debug, Clone)] +pub enum Atom<'a> { + Variable(Variable<'a>), + String(Slice<'a>), + Number(f64), + BigInt(), + BigFloat(), + Regex(Slice<'a>), +} + +#[derive(Debug, Clone)] +pub enum RulePattern<'a> { + Expression(Expr<'a>), + Range(Expr<'a>, Expr<'a>), +} + +#[derive(Debug, Clone)] +pub enum SpecialPattern { + Begin, + End, + BeginFile, + EndFile, +} + +#[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)] +pub struct Identifier<'a> { + pub namespace: &'a str, + pub literal: &'a str, +} + +#[derive(Debug, Clone, Copy)] +pub enum Variable<'a> { + User(Identifier<'a>), + Nr, + Nf, + Fs, + Rs, + Ofs, + Ors, + Filename, + Argc, + Argv, + Subsep, + Fnr, + Argind, + Ofmt, + Rstart, + Rlength, + Environ, +} + +#[derive(Clone)] +pub enum Expr<'a> { + Leaf(Atom<'a>), + Node(&'a ExprNode<'a>), +} + +pub type Body<'a> = Vec<'a, Statement<'a>>; +pub type Pattern<'a> = Either, SpecialPattern>; + +#[derive(Debug, Clone)] +pub enum ExprNode<'a> { + FunctionCall(Identifier<'a>, Vec<'a, Expr<'a>>), + UnaryOperation(UnaryOperator, Expr<'a>), + BinaryOperation(BinaryOperator, Expr<'a>, Expr<'a>), + PlaceOperation(PlaceOperator, Variable<'a>, Expr<'a>), + Ternary(Expr<'a>, Expr<'a>, Expr<'a>), +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum UnaryOperator { + Record, + Negation, + ToInt, + Negative, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum BinaryOperator { + Concat, + Eq, + NEq, + Gt, + Lt, + LtE, + GtE, + And, + Or, + MatchRegex, + NotMatchRegex, + Add, + Subtract, + Multiply, + Divide, + Raise, + Modulo, + InArray, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum PlaceOperator { + Assignment, + ArrayAccess, + InArray, +} + +/// GNU docs: https://www.gnu.org/software/gawk/manual/html_node/Redirection.html +#[derive(Debug, Clone)] +pub enum Redirection<'a> { + Pipe(&'a str), + Write(&'a str), + Append(&'a str), + /// GNU Extension. + WriteCoprocess(&'a [u8]), + /// GNU Extension; only on `getline`. + ReadCoprocess(&'a [u8]), + WriteVariable(), +} + +#[derive(Clone)] +pub enum Statement<'a> { + Expression(Expr<'a>), + Command { + name: Command, + args: Vec<'a, Expr<'a>>, + redirection: Option>, + }, + If { + condition: Expr<'a>, + then_body: Body<'a>, + else_body: Option>, + }, + While { + condition: Expr<'a>, + then_body: Body<'a>, + }, + DoWhile { + then_body: Body<'a>, + condition: Expr<'a>, + }, + // TODO: Maybe Expr is too restrictive for init & update. + // Maybe a subset of Statement? + For { + init: Option>, + condition: Option>, + update: Option>, + body: Body<'a>, + }, + ForEach { + place: Variable<'a>, + array: Variable<'a>, + body: Body<'a>, + }, + Switch { + scrutinee: Expr<'a>, + branches: Vec<'a, (Atom<'a>, Body<'a>)>, + default: Option<(Body<'a>, usize)>, + }, + Break, + Continue, + Return(Option>), +} + +#[derive(Debug, Clone)] +pub struct Function<'a> { + pub args: Vec<'a, Identifier<'a>>, + pub body: Body<'a>, +} + +#[derive(Debug, Clone)] +pub enum Command { + Print, + Printf, + Getline, + Next, + NextFile, + Exit, +} + +#[derive(Debug, Clone)] +pub enum CommandArity { + Nullary, + Unary, + Variadic, +} + +impl Command { + pub fn arity(&self) -> CommandArity { + match self { + Self::Next | Self::NextFile => CommandArity::Nullary, + Self::Getline | Self::Exit => CommandArity::Unary, + Self::Print | Self::Printf => CommandArity::Variadic, + } + } +} + +impl<'a> Expr<'a> { + pub fn leaf(from: impl Into>) -> Self { + Self::Leaf(from.into()) + } + + pub fn node(op: ExprNode<'a>, arena: &'a Bump) -> Self { + Self::Node(arena.alloc(op)) + } +} + +impl UnaryOperator { + pub fn expr(self, a: Expr<'_>) -> ExprNode<'_> { + ExprNode::UnaryOperation(self, a) + } +} + +impl BinaryOperator { + pub fn expr<'a>(self, a: Expr<'a>, b: Expr<'a>) -> ExprNode<'a> { + ExprNode::BinaryOperation(self, a, b) + } +} + +impl PlaceOperator { + pub fn expr<'a>(self, a: Variable<'a>, b: Expr<'a>) -> ExprNode<'a> { + ExprNode::PlaceOperation(self, a, b) + } +} + +impl Expr<'_> { + pub fn take(&mut self) -> Self { + std::mem::replace(self, Self::Leaf(Atom::Number(0.))) + } +} + +impl<'a> From> for Variable<'a> { + fn from(value: Identifier<'a>) -> Self { + Self::User(value) + } +} + +impl<'a> From> for Atom<'a> { + fn from(value: Variable<'a>) -> Self { + Self::Variable(value) + } +} + +impl From for Atom<'_> { + fn from(value: f64) -> Self { + Self::Number(value) + } +} + +impl<'a> TryFrom<&Token<'a>> for UnaryOperator { + type Error = ParsingError; + + fn try_from(value: &Token<'a>) -> Result { + match value { + Token::Record => Ok(Self::Record), + Token::Negation => Ok(Self::Negation), + Token::Plus => Ok(Self::ToInt), + Token::Minus => Ok(Self::Negative), + _ => Err(ParsingError::UnexpectedToken), + } + } +} + +impl<'a> TryFrom<&Token<'a>> for BinaryOperator { + type Error = ParsingError; + + fn try_from(value: &Token<'a>) -> Result { + match value { + Token::EqualTo => Ok(Self::Eq), + Token::NotEqualTo => Ok(Self::NEq), + Token::GreaterThan => Ok(Self::Gt), + Token::LesserThan => Ok(Self::Lt), + Token::GreaterOrEqualThan => Ok(Self::GtE), + Token::LesserOrEqualThan => Ok(Self::LtE), + Token::Matching => Ok(Self::MatchRegex), + Token::NotMatching => Ok(Self::NotMatchRegex), + Token::Plus => Ok(Self::Add), + Token::Minus => Ok(Self::Subtract), + Token::Star => Ok(Self::Multiply), + Token::Slash => Ok(Self::Divide), + t if t.is_expr_start() => Ok(Self::Concat), + _ => Err(ParsingError::UnexpectedToken), + } + } +} + +impl<'a> TryFrom<&Token<'a>> for PlaceOperator { + type Error = ParsingError; + + fn try_from(value: &Token<'a>) -> Result { + match value { + Token::Assignment + | Token::PlusAssign + | Token::MinusAssign + | Token::StarAssign + | Token::SlashAssign + | Token::CaretAssign + | Token::PercentAssign => Ok(Self::Assignment), + Token::OpenBracket => Ok(Self::ArrayAccess), + Token::In => Ok(Self::InArray), + _ => Err(ParsingError::UnexpectedToken), + } + } +} + +impl BinaryOperator { + pub fn unfold(token: &Token) -> Option { + match token { + Token::PlusAssign => Some(Self::Add), + Token::MinusAssign => Some(Self::Subtract), + Token::StarAssign => Some(Self::Multiply), + Token::SlashAssign => Some(Self::Divide), + Token::PercentAssign => Some(Self::Modulo), + Token::CaretAssign => Some(Self::Raise), + _ => None, + } + } + + pub fn unfold_prefix(token: &Token<'_>) -> Option<(Self, u8)> { + match token { + Token::Increment => Some((Self::Add, binding_powers::BP_INC_DEC)), + Token::Decrement => Some((Self::Subtract, binding_powers::BP_INC_DEC)), + _ => None, + } + } + pub fn unfold_suffix(token: &Token<'_>) -> Option<(Self, Self, u8)> { + match token { + Token::Increment => Some((Self::Add, Self::Subtract, binding_powers::BP_INC_DEC)), + Token::Decrement => Some((Self::Subtract, Self::Add, binding_powers::BP_INC_DEC)), + _ => None, + } + } +} + +pub struct Ternary; + +mod binding_powers { + pub const BP_GROUPING: u8 = 32; // [, (, ), ] + pub const BP_RECORD: u8 = 30; // $ + pub const BP_INC_DEC: u8 = 28; // ++, -- + pub const BP_RAISE: (u8, u8) = (27, 26); // ^, ** + pub const BP_UNARY: u8 = 24; // +, -, ! + pub const BP_MULTI: (u8, u8) = (20, 21); // *, /, % + pub const BP_ADDITION: (u8, u8) = (18, 19); // +, - + pub const BP_CONCAT: (u8, u8) = (16, 17); + pub const BP_CMP: (u8, u8) = (14, 15); // < <= == != > >= >> | |& + pub const BP_MATCH: (u8, u8) = (12, 13); // ~, !~ + pub const BP_IN: (u8, u8) = (10, 11); // in + pub const BP_AND: (u8, u8) = (8, 9); // && + pub const BP_OR: (u8, u8) = (6, 7); // || + pub const BP_TERNARY: (u8, u8) = (4, 3); // ?, : + pub const BP_ASSIGN: (u8, u8) = (2, 1); // =, {+, -, *, /, %, ^, **}= +} + +pub trait BindingPower { + type Bp; + fn binding_power(&self) -> Self::Bp; +} + +impl BindingPower for BinaryOperator { + type Bp = (u8, u8); + fn binding_power(&self) -> Self::Bp { + match self { + Self::Concat => binding_powers::BP_CONCAT, + Self::Eq | Self::NEq | Self::Gt | Self::Lt | Self::LtE | Self::GtE => { + binding_powers::BP_CMP + } + Self::And => binding_powers::BP_AND, + Self::Or => binding_powers::BP_OR, + Self::MatchRegex | Self::NotMatchRegex => binding_powers::BP_MATCH, + Self::Add => binding_powers::BP_ADDITION, + Self::Subtract => binding_powers::BP_ADDITION, + Self::Multiply => binding_powers::BP_MULTI, + Self::Divide => binding_powers::BP_MULTI, + Self::Raise => binding_powers::BP_RAISE, + Self::Modulo => binding_powers::BP_RAISE, + Self::InArray => binding_powers::BP_IN, + } + } +} + +impl BindingPower for PlaceOperator { + type Bp = (u8, u8); + fn binding_power(&self) -> Self::Bp { + match self { + Self::Assignment => binding_powers::BP_ASSIGN, + Self::ArrayAccess => (binding_powers::BP_GROUPING, 0), + Self::InArray => binding_powers::BP_IN, + } + } +} + +impl BindingPower for UnaryOperator { + type Bp = u8; + fn binding_power(&self) -> Self::Bp { + match self { + Self::Record => binding_powers::BP_RECORD, + Self::Negation => binding_powers::BP_UNARY, + Self::ToInt => binding_powers::BP_UNARY, + Self::Negative => binding_powers::BP_UNARY, + } + } +} + +impl BindingPower for Ternary { + type Bp = (u8, u8); + + fn binding_power(&self) -> Self::Bp { + binding_powers::BP_TERNARY + } +} + +impl Debug for Expr<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Leaf(atom) => write!(f, "{atom:?}"), + Self::Node(expr) => match expr { + ExprNode::FunctionCall(ident, args) => { + write!(f, "(call Identifier({ident:?})")?; + for arg in args { + write!(f, " {arg:?}")?; + } + write!(f, ")") + } + ExprNode::UnaryOperation(op, a) => write!(f, "({op:?} {a:?})"), + ExprNode::BinaryOperation(op, a, b) => write!(f, "({op:?} {a:?} {b:?})"), + ExprNode::PlaceOperation(op, a, b) => write!(f, "({op:?} {a:?} {b:?})"), + ExprNode::Ternary(a, b, c) => write!(f, "(?: {a:?} {b:?} {c:?})"), + }, + } + } +} + +struct ListLispFmt<'a, T: Debug>(&'a [T]); +struct ListLispCasesFmt<'a, I: Debug, T: Debug>(&'a [(I, Vec<'a, T>)]); + +impl Debug for Statement<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Statement::Expression(expr) => write!(f, "(body {expr:?})"), + Self::Command { + name, + args, + redirection, + } => { + if let Some(rx) = redirection { + write!(f, "(redir {rx:?} ({name:?}{:?}))", ListLispFmt(args)) + } else { + write!(f, "({name:?}{:?})", ListLispFmt(args)) + } + } + Self::If { + condition, + then_body, + else_body, + } => { + if let Some(else_body) = else_body { + write!( + f, + "(if {condition:?} (body{:?}) Some(body{:?}))", + ListLispFmt(then_body), + ListLispFmt(else_body) + ) + } else { + write!( + f, + "(if {condition:?} (body{:?}) None)", + ListLispFmt(then_body) + ) + } + } + Self::While { + condition, + then_body, + } => write!( + f, + "(while {condition:?} (body{:?}))", + ListLispFmt(then_body) + ), + Self::DoWhile { + then_body, + condition, + } => write!( + f, + "(do-while (body{:?}) {condition:?})", + ListLispFmt(then_body) + ), + Self::For { + init, + condition, + update, + body, + } => { + write!( + f, + "(for {init:?} {condition:?} {update:?} (body{:?}))", + ListLispFmt(body) + ) + } + Self::ForEach { place, array, body } => { + write!( + f, + "(for-each {place:?} {array:?} (body{:?}))", + ListLispFmt(body) + ) + } + Self::Switch { + scrutinee, + branches, + default, + } => { + if let Some((dx, i)) = default { + write!( + f, + "(switch {scrutinee:?} (cases{:?}) (body{:?}) {i})", + ListLispCasesFmt(branches), + ListLispFmt(dx) + ) + } else { + write!( + f, + "(switch {scrutinee:?} (cases{:?}))", + ListLispCasesFmt(branches) + ) + } + } + Self::Continue => write!(f, "(continue)"), + Self::Break => write!(f, "(break)"), + Self::Return(expr) => write!(f, "(return {expr:?})"), + } + } +} + +impl Debug for ListLispFmt<'_, T> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for e in self.0 { + write!(f, " {e:?}")?; + } + Ok(()) + } +} + +impl Debug for ListLispCasesFmt<'_, I, T> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for (i, e) in self.0 { + write!(f, " (case {i:?} (eval{:?})", ListLispFmt(e))?; + } + Ok(()) + } +} + +impl Debug for Identifier<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}::{}", self.namespace, self.literal) + } +} diff --git a/parser/src/lex.rs b/parser/src/lex.rs new file mode 100644 index 0000000..f63bc88 --- /dev/null +++ b/parser/src/lex.rs @@ -0,0 +1,204 @@ +use std::iter::Peekable; + +use lexer::{LexingError, Logos, Token}; + +use crate::{ + ParsingError, + ast::{Command, SpecialPattern}, +}; + +#[derive(Debug)] +pub struct Lexer<'a> { + inner: Peekable>, + // source: &'a [u8], +} + +impl<'a> Lexer<'a> { + pub fn new(source: &'a [u8]) -> Self { + Self { + inner: Token::lexer(source).peekable(), + // source, + } + } + + pub fn peek(&mut self) -> Option<&::Item> { + self.inner.peek() + } + + pub fn peek_with(&mut self, f: impl FnOnce(&Token<'a>) -> bool) -> bool { + self.peek().is_some_and(|r| r.as_ref().is_ok_and(f)) + } + + pub fn peek_is(&mut self, b: &Token<'a>) -> bool { + self.peek() + .is_some_and(|r| r.as_ref().is_ok_and(|t| t == b)) + } + + pub fn expect(&mut self, expected: &Token) -> super::Result> { + match self.next() { + Some(Ok(tok)) if expected == &tok => Ok(tok), + Some(Ok(_)) => Err(ParsingError::UnexpectedToken), + Some(err @ Err(_)) => err.map_err(Into::into), + None => todo!(), + } + } + + pub fn expect_with( + &mut self, + expected: impl FnOnce(&Token<'a>) -> bool, + ) -> super::Result> { + match self.next() { + Some(Ok(tok)) if expected(&tok) => Ok(tok), + Some(Ok(_)) => Err(ParsingError::UnexpectedToken), + Some(err @ Err(_)) => err.map_err(Into::into), + None => todo!(), + } + } + + pub fn expect_identifier(&mut self) -> super::Result> { + let Token::Identifier(name) = self.expect_with(|t| matches!(t, Token::Identifier(_)))? + else { + unreachable!() + }; + Ok(name) + } + + pub fn consume(&mut self, token: &Token) -> bool { + if let Some(Ok(next)) = self.peek() + && next == token + { + self.next(); + true + } else { + false + } + } + + pub fn consume_with(&mut self, f: impl FnOnce(&Token<'a>) -> bool) -> bool { + if let Some(Ok(next)) = self.peek() + && f(next) + { + self.next(); + true + } else { + false + } + } + + pub fn next_if( + &mut self, + f: impl FnOnce(&::Item) -> bool, + ) -> Option<::Item> { + self.inner.next_if(f) + } + + pub fn expect_next(&mut self) -> super::Result> { + match self.next() { + None => Err(ParsingError::LexingError(LexingError::UnexpectedEof)), + Some(Ok(tok)) => Ok(tok), + Some(Err(err)) => Err(ParsingError::LexingError(err)), + } + } + + pub fn expect_peek(&mut self) -> super::Result<&Token<'a>> { + match self.peek() { + None => Err(ParsingError::LexingError(LexingError::UnexpectedEof)), + Some(Ok(tok)) => Ok(tok), + Some(Err(err)) => Err(ParsingError::LexingError(err.clone())), + } + } +} + +impl<'a> Iterator for Lexer<'a> { + type Item = Result, LexingError>; + + fn next(&mut self) -> Option { + self.inner.next() + } +} + +pub trait TokenExt { + fn is_prefix_op(&self) -> bool; + fn is_atom(&self) -> bool; + fn is_expr_start(&self) -> bool; + fn is_pattern_start(&self) -> bool; + fn maps_to_command(&self) -> Option; + fn maps_to_special_pat(&self) -> Option; + fn is_stmnt_end(&self) -> bool; + fn is_stmnt_or_block_end(&self) -> bool; + fn is_brace(&self) -> bool; +} + +impl TokenExt for Token<'_> { + fn is_prefix_op(&self) -> bool { + matches!( + self, + Token::Increment + | Token::Decrement + | Token::Record + | Token::Negation + | Token::Minus + | Token::Plus + | Token::Getline + ) + } + fn is_atom(&self) -> bool { + matches!( + self, + Token::Number(_) + | Token::String(_) + | Token::Regex(_) + | Token::Identifier(_) + | Token::NrVariable + | Token::NfVariable + | Token::FsVariable + | Token::RsVariable + | Token::OfsVariable + | Token::OrsVariable + | Token::FilenameVariable + | Token::ArgcVariable + | Token::ArgvVariable + | Token::SubsepVariable + | Token::FnrVariable + | Token::OfmtVariable + | Token::RstartVariable + | Token::RlengthVariable + | Token::EnvironVariable + ) + } + fn is_expr_start(&self) -> bool { + self.is_atom() || self.is_prefix_op() + } + fn is_pattern_start(&self) -> bool { + self.is_expr_start() || self.maps_to_special_pat().is_some() + } + fn maps_to_command(&self) -> Option { + match self { + Token::Print => Some(Command::Print), + Token::Printf => Some(Command::Printf), + Token::Getline => Some(Command::Getline), + Token::Next => Some(Command::Next), + Token::NextFile => Some(Command::NextFile), + Token::Exit => Some(Command::Exit), + _ => None, + } + } + fn maps_to_special_pat(&self) -> Option { + match self { + Self::BeginPattern => Some(SpecialPattern::Begin), + Self::EndPattern => Some(SpecialPattern::End), + Self::BeginFilePattern => Some(SpecialPattern::BeginFile), + Self::EndFilePattern => Some(SpecialPattern::EndFile), + _ => None, + } + } + fn is_stmnt_end(&self) -> bool { + matches!(self, Token::Newline | Token::Semicolon) + } + fn is_stmnt_or_block_end(&self) -> bool { + self.is_stmnt_end() || self == &Token::ClosedBrace + } + fn is_brace(&self) -> bool { + matches!(self, Token::OpenBrace | Token::ClosedBrace) + } +} diff --git a/parser/src/lib.rs b/parser/src/lib.rs new file mode 100644 index 0000000..17c5603 --- /dev/null +++ b/parser/src/lib.rs @@ -0,0 +1,696 @@ +mod ast; +mod lex; + +use std::{fmt::Debug, mem::replace}; + +use bumpalo::{Bump, collections::Vec, vec}; +use either::Either::{Left, Right}; +use hashbrown::HashMap; +use lexer::{LexingError, Token}; +use thiserror::Error; + +pub use crate::ast::Ast; +pub use crate::lex::Lexer; +use crate::{ + ast::{ + Atom, BinaryOperator, BindingPower, Body, Command, CommandArity, Expr, ExprNode, Function, + Identifier, Pattern, PlaceOperator, Rule, RulePattern, SpecialPattern, Statement, Ternary, + UnaryOperator, Variable, + }, + lex::TokenExt, +}; + +type Result = std::result::Result; + +pub struct Parser<'a> { + ast: Ast<'a>, + arena: &'a Bump, + preprocessor: Preprocessor, + namespace: &'a str, + concurrent: bool, +} + +#[derive(Debug, Error, Clone)] +pub enum ParsingError { + #[error("")] + LexingError(LexingError), + #[error("Unclosed scope.")] + UnclosedScope, + #[error("Unexpected token")] + UnexpectedToken, + #[error("Duplicated argument")] + DuplicatedArgument, +} + +impl From for ParsingError { + fn from(value: LexingError) -> Self { + Self::LexingError(value) + } +} + +impl<'a> Parser<'a> { + #[tracing::instrument] + pub fn new(arena: &'a Bump) -> Self { + Self { + ast: Ast::new(arena), + arena, + preprocessor: Preprocessor {}, + namespace: "awk", + concurrent: false, + } + } + + #[tracing::instrument] + pub fn parse(&mut self, lex: &mut Lexer<'a>, awk_namespace: bool) -> Result<&Ast<'a>> { + // Expects: + // * Directive + // * Namespace: Either handle here or in interpreter; idk. + // * Include: recursively lex & parse the filename. + // * Concurrent: Pass on to interpreter. + // * Load: Pass on to interpreter. + // * Pattern (Expression) + // * Expects brackets afterwards (body) or a newline (default). + // * Action (Statement) + // * Expects a newline afterwards; inserts default pattern. + while let Some(tok) = lex.peek() { + if tok.as_ref().is_ok_and(Token::is_pattern_start) { + match self.parse_pattern(lex)? { + Left(rule_pattern) => { + let body = lex.peek_is(&Token::OpenBrace).then(|| self.parse_body(lex)); + self.add_rule(Rule { + pattern: Some(rule_pattern), + actions: body.transpose()?, + }); + } + Right(special_pattern) => { + let body = self.parse_body(lex)?; + match special_pattern { + SpecialPattern::Begin => &mut self.ast.begin, + SpecialPattern::End => &mut self.ast.end, + SpecialPattern::BeginFile => &mut self.ast.begin_file, + SpecialPattern::EndFile => &mut self.ast.end_file, + } + .push(body); + } + } + } else if lex.peek_is(&Token::OpenBrace) { + let actions = Some(self.parse_body(lex)?); + self.add_rule(Rule { + pattern: None, + actions, + }); + } else { + match lex.expect_next()? { + Token::LoadDirective(lib) => { + self.ast.loads.push(lib); + lex.expect_with(Token::is_stmnt_end)?; + } + Token::IncludeDirective(path) => { + let old_namespace = self.namespace; + let content = self.preprocessor.include_in(path.as_ref(), self.arena); + self.parse(&mut Lexer::new(content), true)?; + lex.expect_with(Token::is_stmnt_end)?; + self.namespace = old_namespace; + } + Token::NsIncludeDirective(path) => { + let old_namespace = self.namespace; + let content = self.preprocessor.include_in(path.as_ref(), self.arena); + self.parse(&mut Lexer::new(content), false)?; + lex.expect_with(Token::is_stmnt_end)?; + self.namespace = old_namespace; + } + Token::NamespaceDirective(namespace) => { + self.namespace = namespace; + lex.expect_with(Token::is_stmnt_end)?; + } + Token::ConcurrentDirective => { + if lex.peek_with(|t| t.maps_to_special_pat().is_some()) || self.concurrent { + return Err(ParsingError::UnexpectedToken); + } + self.concurrent = true; + } + Token::Function => self.parse_function(lex)?, + Token::Newline | Token::Semicolon if self.concurrent => { + return Err(ParsingError::UnexpectedToken); + } + Token::Newline | Token::Semicolon => {} + x => unimplemented!("{x:?}"), + } + } + } + Ok(&self.ast) + } + + /// Parses up until `{`. + #[tracing::instrument] + fn parse_pattern(&mut self, lex: &mut Lexer<'a>) -> Result> { + match lex.expect_peek()? { + Token::BeginPattern => Ok(Right(SpecialPattern::Begin)), + Token::EndPattern => Ok(Right(SpecialPattern::End)), + Token::BeginFilePattern => Ok(Right(SpecialPattern::BeginFile)), + Token::EndFilePattern => Ok(Right(SpecialPattern::EndFile)), + _ => { + let expr = self.parse_expression(lex)?; + Ok(Left(if lex.consume(&Token::Comma) { + let expr_end = self.parse_expression(lex)?; + RulePattern::Range(expr, expr_end) + } else { + RulePattern::Expression(expr) + })) + } + } + } + + /// Parses up until `}`. Inserts a lone print statement if none. + #[tracing::instrument] + fn parse_body(&mut self, lex: &mut Lexer<'a>) -> Result> { + lex.expect(&Token::OpenBrace)?; + let mut body = Vec::new_in(self.arena); + let mut depth = 0; + + loop { + if lex.peek_with(|tok| tok.is_stmnt_end() || tok.is_brace()) { + match lex.expect_next()? { + Token::ClosedBrace => { + if depth == 0 { + break Ok(body); + } + depth -= 1; + } + Token::OpenBrace => { + depth += 1; + } + _ => {} + } + } else if lex.peek().is_some() { + body.push(self.parse_statement(lex)?); + } else { + break Err(ParsingError::UnclosedScope); + } + } + } + + #[tracing::instrument] + fn parse_statement(&mut self, lex: &mut Lexer<'a>) -> Result> { + let statement = if lex.expect_peek()?.is_expr_start() { + self.parse_expression(lex).map(Statement::Expression)? + } else { + match lex.expect_next()? { + tok if let Some(name) = tok.maps_to_command() => self.parse_command(lex, name)?, + Token::If => { + let condition = self.parse_parenthesized_expr(lex)?; + let then_body = self.parse_statement_body(lex)?; + let else_body = lex + .consume(&Token::Else) + .then(|| self.parse_statement_body(lex)) + .transpose()?; + Statement::If { + condition, + then_body, + else_body, + } + } + Token::For => { + // FIXME(trivial): parser differential w/ GNU: they treat + // for (ident in ident; expr; expr) as a syntax error. + // It seems like a bug to me. + lex.expect(&Token::OpenParent)?; + let init = (!lex.consume(&Token::Semicolon)) + .then(|| self.parse_expression(lex)) + .transpose()?; + if lex.consume(&Token::Semicolon) || init.is_none() { + let condition = self.parse_for_fragment::(lex)?; + let update = self.parse_for_fragment::(lex)?; + let body = self.parse_statement_body(lex)?; + Statement::For { + init, + condition, + update, + body, + } + } else { + let Some(Expr::Node(ExprNode::PlaceOperation( + PlaceOperator::InArray, + place, + Expr::Leaf(Atom::Variable(array)), + ))) = init + else { + return Err(ParsingError::UnexpectedToken); + }; + lex.expect(&Token::ClosedParent)?; + let body: Vec<'_, Statement<'_>> = self.parse_statement_body(lex)?; + Statement::ForEach { + place: *place, + array: *array, + body, + } + } + } + Token::Switch => { + let scrutinee = self.parse_parenthesized_expr(lex)?; + lex.expect(&Token::OpenBrace)?; + let mut default = None; + let mut branches = Vec::new_in(self.arena); + let mut case = None; + let mut body = Vec::new_in(self.arena); + + while !lex.consume(&Token::ClosedBrace) { + if lex.peek_is(&Token::Case) { + match case.take() { + Some(Right(())) => { + default = Some(( + replace(&mut body, Vec::new_in(self.arena)), + branches.len(), + )); + } + Some(Left(atom)) => branches + .push((atom, replace(&mut body, Vec::new_in(self.arena)))), + _ => {} + } + case = Some(Left(self.parse_case(lex)?)); + } else if lex.consume(&Token::Default) { + lex.expect(&Token::Colon)?; + if default.is_some() || matches!(case, Some(Right(()))) { + return Err(ParsingError::UnexpectedToken); + } else if let Some(Left(atom)) = case { + branches.push((atom, replace(&mut body, Vec::new_in(self.arena)))); + } + case = Some(Right(())); + } else { + if case.is_none() { + return Err(ParsingError::UnexpectedToken); + } + let statement = self.parse_statement(lex)?; + body.push(statement); + } + } + match case.take() { + Some(Right(())) => default = Some((body, branches.len())), + Some(Left(atom)) => branches.push((atom, body)), + _ => {} + } + + Statement::Switch { + scrutinee, + branches, + default, + } + } + Token::While => { + let condition = self.parse_parenthesized_expr(lex)?; + let then_body = self.parse_statement_body(lex)?; + Statement::While { + condition, + then_body, + } + } + Token::Do => { + let then_body = self.parse_body(lex)?; + lex.expect(&Token::While)?; + let condition = self.parse_parenthesized_expr(lex)?; + Statement::DoWhile { + then_body, + condition, + } + } + Token::Break => Statement::Break, + Token::Continue => Statement::Continue, + Token::Return => Statement::Return( + (!lex.peek_with(Token::is_stmnt_or_block_end)) + .then(|| self.parse_expression(lex)) + .transpose()?, + ), + a => todo!("{a:?}"), + } + }; + + lex.consume_with(Token::is_stmnt_end); + Ok(statement) + } + + #[tracing::instrument] + fn parse_parenthesized_expr(&mut self, lex: &mut Lexer<'a>) -> Result> { + lex.expect(&Token::OpenParent)?; + let expr = self.parse_expression(lex)?; + lex.expect(&Token::ClosedParent)?; + Ok(expr) + } + + #[tracing::instrument] + fn parse_for_fragment( + &mut self, + lex: &mut Lexer<'a>, + ) -> Result>> { + let next = if END { + Token::ClosedParent + } else { + Token::Semicolon + }; + if lex.consume(&next) { + Ok(None) + } else { + let expr = self.parse_expression(lex)?; + lex.expect(&next)?; + Ok(Some(expr)) + } + } + + #[tracing::instrument] + fn parse_statement_body(&mut self, lex: &mut Lexer<'a>) -> Result> { + if lex.peek_is(&Token::OpenBrace) { + self.parse_body(lex) + } else { + Ok(vec![in self.arena; self.parse_statement(lex)?]) + } + } + + #[tracing::instrument] + fn parse_case(&mut self, lex: &mut Lexer<'a>) -> Result> { + lex.expect(&Token::Case)?; + let value = self.parse_atom(lex)?; + lex.expect(&Token::Colon)?; + match value { + Atom::Variable(_) => Err(ParsingError::UnexpectedToken), + _ => Ok(value), + } + } + + #[tracing::instrument] + fn parse_command(&mut self, lex: &mut Lexer<'a>, command: Command) -> Result> { + Ok(Statement::Command { + args: match command.arity() { + CommandArity::Nullary => vec![in self.arena], + // TODO: Handle missing argument. + CommandArity::Unary => { + if lex.peek_with(Token::is_stmnt_or_block_end) { + vec![in self.arena] + } else { + vec![in self.arena; self.parse_expression(lex)?] + } + } + + CommandArity::Variadic => { + self.parse_arguments(lex, |t| t.is_stmnt_end() || t == &Token::ClosedBrace)? + } + }, + name: command, + redirection: None, + }) + } + + /// Parses arguments to command or function calls; consumes to the end of + /// the argument list or short-circuits with `delimiter` if empty. + fn parse_arguments( + &mut self, + lex: &mut Lexer<'a>, + delimiter: impl Fn(&Token<'a>) -> bool, + ) -> Result>> { + let mut arguments = Vec::new_in(self.arena); + if lex.peek_with(&delimiter) { + return Ok(arguments); + } + + arguments.push(self.parse_expression(lex)?); + while lex.consume(&Token::Comma) { + arguments.push(self.parse_expression(lex)?); + } + Ok(arguments) + } + + #[tracing::instrument] + fn parse_function(&mut self, lex: &mut Lexer<'a>) -> Result<()> { + let name = lex.expect_identifier()?.qualify(self.namespace); + let args = self.parse_signature(lex)?; + let body = self.parse_body(lex)?; + + self.ast.functions.insert(name, Function { args, body }); + Ok(()) + } + + #[tracing::instrument] + fn parse_signature(&mut self, lex: &mut Lexer<'a>) -> Result>> { + let mut args = Vec::new_in(self.arena); + lex.expect(&Token::OpenParent)?; + + if lex.consume(&Token::ClosedParent) { + return Ok(args); + } + + loop { + let name = lex.expect_identifier()?.qualify(self.namespace); + // Linear search is fine for the numbers we are working with. + if args.iter().any(|a| a == &name) { + return Err(ParsingError::DuplicatedArgument); + } + args.push(name); + + if !lex.consume(&Token::Comma) { + lex.expect(&Token::ClosedParent)?; + break; + } + } + Ok(args) + } + + #[tracing::instrument] + fn parse_expression(&mut self, lex: &mut Lexer<'a>) -> Result> { + self.parse_pratt_fragment(lex, 0) + } + + #[tracing::instrument] + fn parse_pratt_fragment(&mut self, lex: &mut Lexer<'a>, min_bp: u8) -> Result> { + // TODO: getline expressions https://www.gnu.org/software/gawk/manual/html_node/Getline_002fVariable.html + let mut lhs = if lex.consume(&Token::OpenParent) { + let inner = self.parse_expression(lex)?; + + lex.expect(&Token::ClosedParent)?; + inner + } else if lex.peek_with(Token::is_prefix_op) { + let next = lex.expect_next()?; + if let Some((op, bp)) = BinaryOperator::unfold_prefix(&next) { + let Expr::Leaf(Atom::Variable(rhs)) = self.parse_pratt_fragment(lex, bp)? else { + return Err(ParsingError::UnexpectedToken); + }; + Expr::node( + PlaceOperator::Assignment.expr( + rhs, + Expr::node(op.expr(Expr::leaf(rhs), Expr::leaf(1.)), self.arena), + ), + self.arena, + ) + } else if let Ok(op) = UnaryOperator::try_from(&next) { + let rhs = self.parse_pratt_fragment(lex, op.binding_power())?; + + Expr::node(op.expr(rhs), self.arena) + } else { + return Err(ParsingError::UnexpectedToken); + } + } else if lex.peek_with(|t| matches!(t, Token::FunctionCall(_))) { + self.parse_function_call(lex)? + } else { + Expr::leaf(self.parse_atom(lex)?) + }; + + while let Some(next) = lex.peek() { + let next = next.as_ref().map_err(Clone::clone)?; + + if let Ok(op) = BinaryOperator::try_from(next) + && !matches!(next, Token::Increment | Token::Decrement) + { + let (left_bp, right_bp) = op.binding_power(); + + if left_bp < min_bp { + break; + } + lex.consume_with(|_| op != BinaryOperator::Concat); + + let rhs = self.parse_pratt_fragment(lex, right_bp)?; + lhs = Expr::node(op.expr(lhs, rhs), self.arena); + } else if let Ok(op) = PlaceOperator::try_from(next) { + let (left_bp, right_bp) = op.binding_power(); + let Expr::Leaf(Atom::Variable(var)) = lhs.take() else { + return Err(ParsingError::UnexpectedToken); + }; + + if left_bp < min_bp { + break; + } + let token_op = lex.expect_next()?; + + let mut rhs = self.parse_pratt_fragment(lex, right_bp)?; + if let Some(op) = BinaryOperator::unfold(&token_op) { + rhs = Expr::node(op.expr(Expr::leaf(var), rhs), self.arena); + } else if op == PlaceOperator::ArrayAccess { + while lex.consume(&Token::Comma) { + rhs = Expr::node( + BinaryOperator::Concat.expr( + rhs, + Expr::node( + BinaryOperator::Concat.expr( + Expr::leaf(Variable::Subsep), + self.parse_expression(lex)?, + ), + self.arena, + ), + ), + self.arena, + ); + } + lex.expect(&Token::ClosedBracket)?; + } + lhs = Expr::node(op.expr(var, rhs), self.arena); + } else if next == &Token::QuestionMark { + let (left_bp, right_bp) = Ternary.binding_power(); + if left_bp < min_bp { + break; + } + lex.next(); + let then_branch = self.parse_pratt_fragment(lex, right_bp)?; + lex.expect(&Token::Colon)?; + let else_branch = self.parse_pratt_fragment(lex, right_bp)?; + lhs = Expr::node(ExprNode::Ternary(lhs, then_branch, else_branch), self.arena); + } else if let Some((operation, reciprocal, bp)) = BinaryOperator::unfold_suffix(next) { + let Expr::Leaf(Atom::Variable(rhs)) = lhs else { + return Err(ParsingError::UnexpectedToken); + }; + if bp < min_bp { + break; + } + lex.next(); + lhs = Expr::node( + operation.expr( + Expr::node( + PlaceOperator::Assignment.expr( + rhs, + Expr::node( + reciprocal.expr(Expr::leaf(rhs), Expr::leaf(1.)), + self.arena, + ), + ), + self.arena, + ), + Expr::leaf(1.), + ), + self.arena, + ); + } else { + break; + } + } + Ok(lhs) + } + + #[tracing::instrument] + fn add_rule(&mut self, rule: Rule<'a>) { + if self.concurrent { + self.concurrent = false; + &mut self.ast.concurrent + } else { + &mut self.ast.rules + } + .push(rule); + } + + #[tracing::instrument] + fn parse_function_call(&mut self, lex: &mut Lexer<'a>) -> Result> { + let Token::FunctionCall(ident) = lex.expect_next()? else { + return Err(ParsingError::UnexpectedToken); + }; + let expr = ExprNode::FunctionCall( + ident.qualify(self.namespace), + self.parse_arguments(lex, |t| t == &Token::ClosedParent)?, + ); + lex.expect(&Token::ClosedParent)?; + Ok(Expr::node(expr, self.arena)) + } + + #[tracing::instrument] + fn parse_atom(&self, lex: &mut Lexer<'a>) -> Result> { + match lex.expect_next()? { + Token::Number(n) => Ok(Atom::Number(n)), + Token::String(s) => Ok(Atom::String(s)), + Token::Regex(r) => Ok(Atom::Regex(r)), + Token::Identifier(a) => Ok(Atom::Variable(a.qualify(self.namespace).into())), + Token::NrVariable => Ok(Variable::Nr.into()), + Token::NfVariable => Ok(Variable::Nf.into()), + Token::FsVariable => Ok(Variable::Fs.into()), + Token::RsVariable => Ok(Variable::Rs.into()), + Token::OfsVariable => Ok(Variable::Ofs.into()), + Token::OrsVariable => Ok(Variable::Ors.into()), + Token::FilenameVariable => Ok(Variable::Filename.into()), + Token::ArgcVariable => Ok(Variable::Argc.into()), + Token::ArgvVariable => Ok(Variable::Argv.into()), + Token::SubsepVariable => Ok(Variable::Subsep.into()), + Token::FnrVariable => Ok(Variable::Fnr.into()), + Token::OfmtVariable => Ok(Variable::Ofmt.into()), + Token::RstartVariable => Ok(Variable::Rstart.into()), + Token::RlengthVariable => Ok(Variable::Rlength.into()), + Token::EnvironVariable => Ok(Variable::Environ.into()), + _ => Err(ParsingError::UnexpectedToken), + } + } +} + +impl<'a> Ast<'a> { + fn new(arena: &'a Bump) -> Self { + Self { + loads: Vec::new_in(arena), + begin: Vec::new_in(arena), + end: Vec::new_in(arena), + begin_file: Vec::new_in(arena), + end_file: Vec::new_in(arena), + rules: Vec::new_in(arena), + concurrent: Vec::new_in(arena), + functions: HashMap::new_in(arena), + } + } +} + +#[derive(Debug)] +struct Preprocessor {} + +impl Preprocessor { + fn include_in<'a: 'b, 'b>(&mut self, _path: &'b [u8], _alloc: &'a Bump) -> &'a [u8] { + todo!() + } +} + +trait IdentifierExt<'a> { + fn qualify(self, namespace: &'a str) -> Identifier<'a> + where + Self: 'a; +} + +impl<'a> IdentifierExt<'a> for lexer::Identifier<'_> { + fn qualify(self, namespace: &'a str) -> Identifier<'a> + where + Self: 'a, + { + let literal = self.literal; + if let Some(namespace) = self.namespace { + Identifier { namespace, literal } + } else { + Identifier { namespace, literal } + } + } +} + +impl Debug for Parser<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + #[derive(Debug)] + #[allow(dead_code)] + struct Parser<'a> { + ast: &'a Ast<'a>, + preprocessor: &'a Preprocessor, + namespace: &'a str, + concurrent: bool, + } + Parser { + ast: &self.ast, + preprocessor: &self.preprocessor, + namespace: self.namespace, + concurrent: self.concurrent, + } + .fmt(f) + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..586cde7 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,37 @@ +// static POSIX: bool = false; + +mod utils; + +use bumpalo::Bump; +use color_eyre::Result; +use parser::{Lexer, Parser}; + +use crate::utils::{ensure_consistent_panic, exit_err}; + +fn main() { + if let Err(e) = ensure_consistent_panic(uu_main) { + exit_err(e) + } +} + +#[tracing::instrument] +fn uu_main() -> Result<()> { + let arg = std::env::args().nth(1).unwrap(); + + let arena = Bump::with_capacity(4000); // 4KB minus metadata-ish + let mut lex = Lexer::new(arg.as_bytes()); + let mut parser = Parser::new(&arena); + let ast = dbg!(parser.parse(&mut lex, true).unwrap()); + let arena2 = Bump::new(); + arena2.alloc_with(|| ast.clone()); + dbg!(arena.chunk_capacity()); + + // for token in lex { + // let Ok(x) = token else { + // return token.map(drop).map_err(color_eyre::Report::from); + // }; + // println!("{x:?}"); + // } + // exit_with(Interpreter.run()) + Ok(()) +} diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..4bb770c --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,71 @@ +use std::fmt::{Debug, Display}; +use std::panic::{UnwindSafe, catch_unwind, set_hook, take_hook}; +use std::process::exit; + +use color_eyre::config::HookBuilder; +use rustix::process::{EXIT_FAILURE, EXIT_SUCCESS}; +use tracing_error::ErrorLayer; +use tracing_subscriber::{EnvFilter, prelude::*}; + +type ExitCode = i32; + +const AWK_PANIC_CODE: ExitCode = 2; + +fn install_abort_hook() { + let run_hook = take_hook(); + + set_hook(Box::new(move |info| { + run_hook(info); + exit(AWK_PANIC_CODE); + })); +} + +#[inline(always)] +fn install_error_hooks() { + tracing_subscriber::registry() + // .with(EnvFilter::from_default_env()) + .with(tracing_subscriber::fmt::layer().compact()) + .with(ErrorLayer::default()) + .init(); + + HookBuilder::default() + .capture_span_trace_by_default(true) + .install() + .unwrap(); +} + +/// Ensures exit code 2 on panics, which is GNU's behavior. Also installs +/// color-eyre's panic hook. +/// https://www.gnu.org/software/gawk/manual/html_node/Exit-Status.html +#[inline(always)] // Hide from stack trace on panics. +pub fn ensure_consistent_panic(f: impl UnwindSafe + FnOnce() -> T) -> T { + install_error_hooks(); + if cfg!(panic = "abort") { + // Prevents core dumps on panic. We _might_ want to carve an exception. + install_abort_hook(); + f() + } else { + // If unwinding is enabled, we catch it before the Rust entry point + // does and exit with our custom exit code. The panic msg is printed. + match catch_unwind(f) { + Ok(x) => x, + Err(_) => exit(AWK_PANIC_CODE), + } + } +} + +/// Exits with a custom exit code or libc's codes, as per POSIX. +pub fn exit_with(res: Result>, impl Display + Debug>) -> ! { + let code = match res { + Ok(Some(x)) => x.into(), + Ok(None) => EXIT_SUCCESS, + Err(e) => exit_err(e), + }; + + exit(code) +} + +pub fn exit_err(err: impl Display + Debug) -> ! { + eprintln!("{err:?}"); + exit(EXIT_FAILURE) +} diff --git a/tests/awk.rs b/tests/awk.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/awk.rs @@ -0,0 +1 @@ +