From 41bf0a218490823fbd85babc4ea387ce3359b8be Mon Sep 17 00:00:00 2001 From: Ross Andrews Date: Sun, 2 Jul 2023 02:10:06 -0500 Subject: [PATCH] Starting to build a Forge parser --- .idea/vulcan-emu.iml | 1 + Cargo.lock | 9 ++ Cargo.toml | 3 +- forge_core/Cargo.toml | 11 ++ forge_core/src/forge.pest | 67 ++++++++ forge_core/src/forge_parser.rs | 277 +++++++++++++++++++++++++++++++++ forge_core/src/lib.rs | 5 + forge_core/src/parse_test.txt | 66 ++++++++ 8 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 forge_core/Cargo.toml create mode 100644 forge_core/src/forge.pest create mode 100644 forge_core/src/forge_parser.rs create mode 100644 forge_core/src/lib.rs create mode 100644 forge_core/src/parse_test.txt diff --git a/.idea/vulcan-emu.iml b/.idea/vulcan-emu.iml index 9f53825..461e447 100644 --- a/.idea/vulcan-emu.iml +++ b/.idea/vulcan-emu.iml @@ -11,6 +11,7 @@ + diff --git a/Cargo.lock b/Cargo.lock index a1ce4a2..9028fe2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -446,6 +446,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "forge_core" +version = "0.1.0" +dependencies = [ + "pest", + "pest_derive", + "vcore", +] + [[package]] name = "fxhash" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index a1c6980..9eb54c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,5 +8,6 @@ members = [ "vtest", "vlua", "vweb", - "vgfx" + "vgfx", + "forge_core" ] \ No newline at end of file diff --git a/forge_core/Cargo.toml b/forge_core/Cargo.toml new file mode 100644 index 0000000..7845cdb --- /dev/null +++ b/forge_core/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "forge_core" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +vcore = { path = "../vcore" } +pest = "2.1.3" +pest_derive = "2.1.0" diff --git a/forge_core/src/forge.pest b/forge_core/src/forge.pest new file mode 100644 index 0000000..b9f6ff9 --- /dev/null +++ b/forge_core/src/forge.pest @@ -0,0 +1,67 @@ +WHITESPACE = _{ " " | "\t" | NEWLINE } +COMMENT = _{ "//" ~ (!"\n" ~ ANY)* } + +name_char = { ASCII_ALPHA_LOWER | ASCII_ALPHA_UPPER | "_" | "$" } +name = @{ name_char ~ (name_char | ASCII_DIGIT)* } + +dec_number = @{ ASCII_NONZERO_DIGIT ~ ASCII_DIGIT* } +neg_number = ${ "-" ~ dec_number } +hex_number = ${ "0x" ~ ASCII_HEX_DIGIT+ } +bin_number = ${ "0b" ~ ASCII_BIN_DIGIT+ } +oct_number = ${ "0o" ~ ASCII_OCT_DIGIT+ } +dec_zero = @{ "0" } +number = { dec_number | hex_number | bin_number | oct_number | dec_zero | neg_number } + +assignment = { lvalue ~ "=" ~ rvalue } +lvalue = { arrayref | name } +rvalue = { expr | string } + +sign = { "+" | "-" } +term_op = { "*" | "/" | "%" } +expr = { term ~ (sign ~ term)* } +term = { val ~ (term_op ~ val)* } +val = { // These are what get evaluated and left on the stack: + number | // Literal numbers + ("(" ~ expr ~ ")") | // Nested parenthesized exprs + // Bare literal ids, fn calls, array references + // (call, or pointer arithmetic, since ids all reference addresses) + // (multiple calls / subscripts aren't allowed; use CALL or collapse to one + // subscript) + call | arrayref | name | + //(name ~ (arglist | subscript)?) | + ("&" ~ name) } // The address for an name + +arrayref = { name ~ subscript } // Subscripting is only literal IDs. Anything else can be pointer arithmetic +call = { name ~ arglist } // Call syntax is only literal IDs. Calling things dynamically is done with a builtin; CALL(foo[3], 1, 2, 3) +arg = { expr | string } +arglist = { "(" ~ (arg ~ ("," ~ arg)*)? ~ ")" } +subscript = { "[" ~ expr ~ "]" } + +escape = @{ "\\" ~ ("t" | "r" | "n" | "0" | "\\" | "\"") } +string_inner = ${ !("\"" | "\\") ~ ANY | escape } +string = ${ "\"" ~ string_inner* ~ "\"" } + +statement = { ((return_stmt | assignment | call | declaration) ~ ";") | conditional | loop_stmt } + +block = { "{" ~ statement* ~ "}" } + +function = { "fn" ~ name ~ annotations? ~ argnames ~ block } +annotations = { "<" ~ (annotation ~ ("," ~ annotation)*) ~ ">" } +annotation = { "inline" | ("org=" ~ number) | ("type=" ~ name) } +argnames = { "(" ~ (name ~ typename? ~ ("," ~ name ~ typename?)*)? ~ ")" } + +program = { (COMMENT | WHITESPACE)? ~ (function | global | struct_decl | const_decl)* ~ EOI } + +return_stmt = { "return" ~ expr } +conditional = { "if" ~ "(" ~ expr ~ ")" ~ block ~ ("else" ~ block)? } +loop_stmt = { while_stmt | repeat } +while_stmt = { "while" ~ "(" ~ expr ~ ")" ~ block } +repeat = { "repeat" ~ "(" ~ expr ~ ")" ~ name ~ block } +declaration = { "var" ~ name ~ typename? ~ ("=" ~ expr)? } +global = { "global" ~ name ~ typename? ~ ";" } +typename = { ":" ~ name } +const_decl = { "const" ~ name ~ "=" ~ (number | string) ~ ";" } + +struct_decl = { "struct" ~ name ~ "{" ~ members ~ "}" } +member = { name ~ typename? ~ ("[" ~ number ~ "]")? } +members = { (member ~ ("," ~ member)*)? } \ No newline at end of file diff --git a/forge_core/src/forge_parser.rs b/forge_core/src/forge_parser.rs new file mode 100644 index 0000000..4625fef --- /dev/null +++ b/forge_core/src/forge_parser.rs @@ -0,0 +1,277 @@ +use pest::Parser; + +mod inner { + #[derive(Parser)] + #[grammar = "forge.pest"] + pub struct ForgeParser; +} + +use inner::*; +use pest::error::{Error, LineColLocation}; +use std::str::FromStr; + +type Pair<'a> = pest::iterators::Pair<'a, Rule>; + +fn parse_i32(pair: Pair) -> i32 { + let first = pair.into_inner().next().unwrap(); + match first.as_rule() { + Rule::dec_number | Rule::dec_zero | Rule::neg_number => { + i32::from_str(first.as_str()).unwrap() + } + Rule::hex_number => i32::from_str_radix(first.as_str().get(2..).unwrap(), 16).unwrap(), + Rule::bin_number => i32::from_str_radix(first.as_str().get(2..).unwrap(), 2).unwrap(), + Rule::oct_number => i32::from_str_radix(first.as_str().get(2..).unwrap(), 8).unwrap(), + _ => panic!("Expected a number, got {}", first.as_str()), + } +} + +/// Create a String containing the string represented by a given pair. Since the pair will +/// reference bytes containing escape sequences, this isn't the same as an &str to the +/// original code; this is a new string translating those escape sequences to their actual +/// bytes. +fn create_string(pair: Pair) -> String { + let mut string = String::with_capacity(pair.as_str().len()); + for inner in pair.into_inner() { + let string_inner = inner.as_str(); + match string_inner { + "\\t" => string.push('\t'), + "\\r" => string.push('\r'), + "\\n" => string.push('\n'), + "\\0" => string.push('\0'), + "\\\\" => string.push('\\'), + "\\\"" => string.push('\"'), + _ => string.push_str(string_inner), + } + } + string +} + +#[derive(Eq, PartialEq, Clone, Debug)] +pub struct ParseError(usize, usize, String); + +impl From> for ParseError { + fn from(err: Error) -> Self { + let message = err.to_string(); + match err.line_col { + LineColLocation::Pos((line, col)) | LineColLocation::Span((line, col), _) => { + Self(line, col, message) + } + } + } +} + +#[derive(Eq, PartialEq, Clone, Debug)] +pub enum Node { + Function { + name: String, + org: Option, + typename: Option, + inline: bool, + }, + Global { + name: String, + typename: Option, + }, + Struct { + name: String, + members: Vec, + }, + Const { + name: String, + value: Option, + string: Option, + }, +} + +#[derive(Eq, PartialEq, Clone, Debug)] +pub struct Member { + name: String, + typename: Option, + size: Option, +} + +pub fn string_to_ast(string: &str) -> Result, ParseError> { + let pairs = ForgeParser::parse(Rule::program, string) + .map_err(ParseError::from)? + .next() + .unwrap() + .into_inner(); + + let mut decls = Vec::new(); + for pair in pairs { + let r = pair.as_rule(); + match r { + Rule::function => {} + Rule::global => decls.push(parse_global(pair)), + Rule::struct_decl => decls.push(parse_struct(pair)), + Rule::const_decl => decls.push(parse_const(pair)), + Rule::EOI => {} // Ignore EOI; we have to capture it but it doesn't do anything + _ => unreachable!(), + } + } + + Ok(decls) +} + +fn parse_global(global: Pair) -> Node { + let mut inner = global.into_inner(); + let name = String::from(inner.next().unwrap().as_str()); + let typename = inner + .next() + .map(|p| String::from(p.into_inner().next().unwrap().as_str())); + Node::Global { name, typename } +} + +fn parse_const(pair: Pair) -> Node { + let mut inner = pair.into_inner(); + let name = String::from(inner.next().unwrap().as_str()); + let value = inner.next().unwrap(); + match value.as_rule() { + Rule::string => Node::Const { + name, + string: Some(String::from(create_string(value))), + value: None, + }, + Rule::number => Node::Const { + name, + string: None, + value: Some(parse_i32(value)), + }, + _ => unreachable!(), + } +} + +fn parse_member(pair: Pair) -> Member { + let mut inner = pair.into_inner(); + let mut member = Member { + name: String::from(inner.next().unwrap().as_str()), + typename: None, + size: None, + }; + for child in inner { + match child.as_rule() { + Rule::number => member.size = Some(parse_i32(child)), + Rule::typename => { + member.typename = Some(String::from(child.into_inner().next().unwrap().as_str())) + } + _ => unreachable!(), + } + } + member +} + +fn parse_struct(pair: Pair) -> Node { + let mut inner = pair.into_inner(); + let name = String::from(inner.next().unwrap().as_str()); + let mut member_pairs = inner.next().unwrap().into_inner(); + let members: Vec<_> = member_pairs.map(|member| parse_member(member)).collect(); + Node::Struct { name, members } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn parse_globals() { + let ast = string_to_ast("global foo;"); + assert_eq!( + ast, + Ok(vec![Node::Global { + name: "foo".into(), + typename: None + }]) + ); + + let ast = string_to_ast("global bar:Sometype;"); + assert_eq!( + ast, + Ok(vec![Node::Global { + name: "bar".into(), + typename: Some("Sometype".into()) + }]) + ); + } + + #[test] + fn parse_consts() { + assert_eq!( + string_to_ast("const a = 123;"), + Ok(vec![Node::Const { + name: "a".into(), + value: Some(123), + string: None + }]) + ); + assert_eq!( + string_to_ast("const a = 0xaa;"), + Ok(vec![Node::Const { + name: "a".into(), + value: Some(0xaa), + string: None + }]) + ); + assert_eq!( + string_to_ast("const a = -7;"), + Ok(vec![Node::Const { + name: "a".into(), + value: Some(-7), + string: None + }]) + ); + assert_eq!( + string_to_ast("const a = \"foo bar\";"), + Ok(vec![Node::Const { + name: "a".into(), + value: None, + string: Some("foo bar".into()) + }]) + ) + } + + #[test] + fn parse_structs() { + assert_eq!( + string_to_ast("struct Point { x, y }"), + Ok(vec![Node::Struct { + name: "Point".into(), + members: vec![ + Member { + name: "x".into(), + typename: None, + size: None + }, + Member { + name: "y".into(), + typename: None, + size: None + }, + ] + }]) + ); + + assert_eq!( + string_to_ast("struct Foo { bar[100] }"), + Ok(vec![Node::Struct { + name: "Foo".into(), + members: vec![Member { + name: "bar".into(), + typename: None, + size: Some(100) + },] + }]) + ); + + assert_eq!( + string_to_ast("struct Foo { bar:Thing[100] }"), + Ok(vec![Node::Struct { + name: "Foo".into(), + members: vec![Member { + name: "bar".into(), + typename: Some("Thing".into()), + size: Some(100) + },] + }]) + ); + } +} diff --git a/forge_core/src/lib.rs b/forge_core/src/lib.rs new file mode 100644 index 0000000..60069fa --- /dev/null +++ b/forge_core/src/lib.rs @@ -0,0 +1,5 @@ +extern crate pest; +#[macro_use] +extern crate pest_derive; + +mod forge_parser; diff --git a/forge_core/src/parse_test.txt b/forge_core/src/parse_test.txt new file mode 100644 index 0000000..fba7209 --- /dev/null +++ b/forge_core/src/parse_test.txt @@ -0,0 +1,66 @@ + // header comment (the initial space is purposeful) + +fn blah() { + // comment + foo = 3; // normal assignment + x = 3 * -2; // unary minus + y = 3 - -3; + blah = foo + 5 * bar / 2; // Complex expr in assignment + arr[3] = 7; // Subscript in lvalue + arr[n * 2 - 1] = foo + 7; // expr in subscript + a[3] = b[70 - foo]; // subscripted rvalues + b = foo * (bar + 3); // expr parens + x1 = something(); // fn calls + something(1); // unary calls, call statements + something(1, foo, bar + 7, ha[0]); // exprs in calls + s = "hello"; // strings + s2 = "hello\n\tthere\0"; + print("bah"); + f = &x; +} + +// fn with args +fn foo(a, b) {} + +// fn with an annotation +fn foo2(x) { return x * x; } + +fn iftest() { + if (blah) { + foo(); + } else { + bar(); + } +} + +fn whiletest() { + while (foo) { + bar(); + } +} + +fn repeatTest(x) { + repeat(x * 3) a { + blah(whatever + a); + } +} + +global foo; +global bar:Thing; + +fn varTest(t:Some) { + var x; + var y = 2 + x; + var z:Thingy; +} + +struct Foo {} +struct Bar { + x, y, name:Str +} +struct Str { + chars[100] +} +const LEN = 100; +const STR = "Hey"; +struct What { foo[100] } \ No newline at end of file