Starting to build a Forge parser

2023-07-02 02:10:06 -05:00
parent 51c690e263
commit 41bf0a2184
8 changed files with 438 additions and 1 deletions
@@ -11,6 +11,7 @@
      <sourceFolder url="file://$MODULE_DIR$/vasm/src" isTestSource="false" />
      <sourceFolder url="file://$MODULE_DIR$/vweb/src" isTestSource="false" />
      <sourceFolder url="file://$MODULE_DIR$/vgfx/src" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/forge_core/src" isTestSource="false" />
      <excludeFolder url="file://$MODULE_DIR$/target" />
    </content>
    <orderEntry type="inheritedJdk" />
@@ -446,6 +446,15 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"

+[[package]]
+name = "forge_core"
+version = "0.1.0"
+dependencies = [
+ "pest",
+ "pest_derive",
+ "vcore",
+]
+
 [[package]]
 name = "fxhash"
 version = "0.2.1"
@@ -8,5 +8,6 @@ members = [
    "vtest",
    "vlua",
    "vweb",
-    "vgfx"
+    "vgfx",
+    "forge_core"
 ]
@@ -0,0 +1,11 @@
+[package]
+name = "forge_core"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+vcore = { path = "../vcore" }
+pest = "2.1.3"
+pest_derive = "2.1.0"
@@ -0,0 +1,67 @@
+WHITESPACE = _{ " " | "\t" | NEWLINE }
+COMMENT = _{ "//" ~ (!"\n" ~ ANY)* }
+
+name_char = { ASCII_ALPHA_LOWER | ASCII_ALPHA_UPPER | "_" | "$" }
+name = @{ name_char ~ (name_char | ASCII_DIGIT)* }
+
+dec_number = @{ ASCII_NONZERO_DIGIT ~ ASCII_DIGIT* }
+neg_number = ${ "-" ~ dec_number }
+hex_number = ${ "0x" ~ ASCII_HEX_DIGIT+ }
+bin_number = ${ "0b" ~ ASCII_BIN_DIGIT+ }
+oct_number = ${ "0o" ~ ASCII_OCT_DIGIT+ }
+dec_zero = @{ "0" }
+number = { dec_number | hex_number | bin_number | oct_number | dec_zero | neg_number }
+
+assignment = { lvalue ~ "=" ~ rvalue }
+lvalue = { arrayref | name }
+rvalue = { expr | string }
+
+sign = { "+" | "-" }
+term_op = { "*" | "/" | "%" }
+expr = { term ~ (sign ~ term)* }
+term = { val ~ (term_op ~ val)* }
+val = { // These are what get evaluated and left on the stack:
+  number | // Literal numbers
+  ("(" ~ expr ~ ")") | // Nested parenthesized exprs
+  // Bare literal ids, fn calls, array references
+  // (call, or pointer arithmetic, since ids all reference addresses)
+  // (multiple calls / subscripts aren't allowed; use CALL or collapse to one
+  // subscript)
+  call | arrayref | name |
+  //(name ~ (arglist | subscript)?) |
+  ("&" ~ name) } // The address for an name
+
+arrayref = { name ~ subscript } // Subscripting is only literal IDs. Anything else can be pointer arithmetic
+call = { name ~ arglist } // Call syntax is only literal IDs. Calling things dynamically is done with a builtin; CALL(foo[3], 1, 2, 3)
+arg = { expr | string }
+arglist = { "(" ~ (arg ~ ("," ~ arg)*)? ~ ")" }
+subscript = { "[" ~ expr ~ "]" }
+
+escape = @{ "\\" ~ ("t" | "r" | "n" | "0" | "\\" | "\"") }
+string_inner = ${ !("\"" | "\\") ~ ANY | escape }
+string = ${ "\"" ~ string_inner* ~ "\"" }
+
+statement = { ((return_stmt | assignment | call | declaration) ~ ";") | conditional | loop_stmt }
+
+block = { "{" ~ statement* ~ "}" }
+
+function = { "fn" ~ name ~ annotations? ~ argnames ~ block }
+annotations = { "<" ~ (annotation ~ ("," ~ annotation)*) ~ ">" }
+annotation = { "inline" | ("org=" ~ number) | ("type=" ~ name) }
+argnames = { "(" ~ (name ~ typename? ~ ("," ~ name ~ typename?)*)? ~ ")" }
+
+program = { (COMMENT | WHITESPACE)? ~ (function | global | struct_decl | const_decl)* ~ EOI }
+
+return_stmt = { "return" ~ expr }
+conditional = { "if" ~ "(" ~ expr ~ ")" ~ block ~ ("else" ~ block)? }
+loop_stmt = { while_stmt | repeat }
+while_stmt = { "while" ~ "(" ~ expr ~ ")" ~ block }
+repeat = { "repeat" ~ "(" ~ expr ~ ")" ~ name ~ block }
+declaration = { "var" ~ name ~ typename? ~ ("=" ~ expr)? }
+global = { "global" ~ name ~ typename? ~ ";" }
+typename = { ":" ~ name }
+const_decl = { "const" ~ name ~ "=" ~ (number | string) ~ ";" }
+
+struct_decl = { "struct" ~ name ~ "{" ~ members ~ "}" }
+member = { name ~ typename? ~ ("[" ~ number ~ "]")? }
+members = { (member ~ ("," ~ member)*)? }
@@ -0,0 +1,277 @@
+use pest::Parser;
+
+mod inner {
+    #[derive(Parser)]
+    #[grammar = "forge.pest"]
+    pub struct ForgeParser;
+}
+
+use inner::*;
+use pest::error::{Error, LineColLocation};
+use std::str::FromStr;
+
+type Pair<'a> = pest::iterators::Pair<'a, Rule>;
+
+fn parse_i32(pair: Pair) -> i32 {
+    let first = pair.into_inner().next().unwrap();
+    match first.as_rule() {
+        Rule::dec_number | Rule::dec_zero | Rule::neg_number => {
+            i32::from_str(first.as_str()).unwrap()
+        }
+        Rule::hex_number => i32::from_str_radix(first.as_str().get(2..).unwrap(), 16).unwrap(),
+        Rule::bin_number => i32::from_str_radix(first.as_str().get(2..).unwrap(), 2).unwrap(),
+        Rule::oct_number => i32::from_str_radix(first.as_str().get(2..).unwrap(), 8).unwrap(),
+        _ => panic!("Expected a number, got {}", first.as_str()),
+    }
+}
+
+/// Create a String containing the string represented by a given pair. Since the pair will
+/// reference bytes containing escape sequences, this isn't the same as an &str to the
+/// original code; this is a new string translating those escape sequences to their actual
+/// bytes.
+fn create_string(pair: Pair) -> String {
+    let mut string = String::with_capacity(pair.as_str().len());
+    for inner in pair.into_inner() {
+        let string_inner = inner.as_str();
+        match string_inner {
+            "\\t" => string.push('\t'),
+            "\\r" => string.push('\r'),
+            "\\n" => string.push('\n'),
+            "\\0" => string.push('\0'),
+            "\\\\" => string.push('\\'),
+            "\\\"" => string.push('\"'),
+            _ => string.push_str(string_inner),
+        }
+    }
+    string
+}
+
+#[derive(Eq, PartialEq, Clone, Debug)]
+pub struct ParseError(usize, usize, String);
+
+impl From<pest::error::Error<Rule>> for ParseError {
+    fn from(err: Error<Rule>) -> Self {
+        let message = err.to_string();
+        match err.line_col {
+            LineColLocation::Pos((line, col)) | LineColLocation::Span((line, col), _) => {
+                Self(line, col, message)
+            }
+        }
+    }
+}
+
+#[derive(Eq, PartialEq, Clone, Debug)]
+pub enum Node {
+    Function {
+        name: String,
+        org: Option<i32>,
+        typename: Option<String>,
+        inline: bool,
+    },
+    Global {
+        name: String,
+        typename: Option<String>,
+    },
+    Struct {
+        name: String,
+        members: Vec<Member>,
+    },
+    Const {
+        name: String,
+        value: Option<i32>,
+        string: Option<String>,
+    },
+}
+
+#[derive(Eq, PartialEq, Clone, Debug)]
+pub struct Member {
+    name: String,
+    typename: Option<String>,
+    size: Option<i32>,
+}
+
+pub fn string_to_ast(string: &str) -> Result<Vec<Node>, ParseError> {
+    let pairs = ForgeParser::parse(Rule::program, string)
+        .map_err(ParseError::from)?
+        .next()
+        .unwrap()
+        .into_inner();
+
+    let mut decls = Vec::new();
+    for pair in pairs {
+        let r = pair.as_rule();
+        match r {
+            Rule::function => {}
+            Rule::global => decls.push(parse_global(pair)),
+            Rule::struct_decl => decls.push(parse_struct(pair)),
+            Rule::const_decl => decls.push(parse_const(pair)),
+            Rule::EOI => {} // Ignore EOI; we have to capture it but it doesn't do anything
+            _ => unreachable!(),
+        }
+    }
+
+    Ok(decls)
+}
+
+fn parse_global(global: Pair) -> Node {
+    let mut inner = global.into_inner();
+    let name = String::from(inner.next().unwrap().as_str());
+    let typename = inner
+        .next()
+        .map(|p| String::from(p.into_inner().next().unwrap().as_str()));
+    Node::Global { name, typename }
+}
+
+fn parse_const(pair: Pair) -> Node {
+    let mut inner = pair.into_inner();
+    let name = String::from(inner.next().unwrap().as_str());
+    let value = inner.next().unwrap();
+    match value.as_rule() {
+        Rule::string => Node::Const {
+            name,
+            string: Some(String::from(create_string(value))),
+            value: None,
+        },
+        Rule::number => Node::Const {
+            name,
+            string: None,
+            value: Some(parse_i32(value)),
+        },
+        _ => unreachable!(),
+    }
+}
+
+fn parse_member(pair: Pair) -> Member {
+    let mut inner = pair.into_inner();
+    let mut member = Member {
+        name: String::from(inner.next().unwrap().as_str()),
+        typename: None,
+        size: None,
+    };
+    for child in inner {
+        match child.as_rule() {
+            Rule::number => member.size = Some(parse_i32(child)),
+            Rule::typename => {
+                member.typename = Some(String::from(child.into_inner().next().unwrap().as_str()))
+            }
+            _ => unreachable!(),
+        }
+    }
+    member
+}
+
+fn parse_struct(pair: Pair) -> Node {
+    let mut inner = pair.into_inner();
+    let name = String::from(inner.next().unwrap().as_str());
+    let mut member_pairs = inner.next().unwrap().into_inner();
+    let members: Vec<_> = member_pairs.map(|member| parse_member(member)).collect();
+    Node::Struct { name, members }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn parse_globals() {
+        let ast = string_to_ast("global foo;");
+        assert_eq!(
+            ast,
+            Ok(vec![Node::Global {
+                name: "foo".into(),
+                typename: None
+            }])
+        );
+
+        let ast = string_to_ast("global bar:Sometype;");
+        assert_eq!(
+            ast,
+            Ok(vec![Node::Global {
+                name: "bar".into(),
+                typename: Some("Sometype".into())
+            }])
+        );
+    }
+
+    #[test]
+    fn parse_consts() {
+        assert_eq!(
+            string_to_ast("const a = 123;"),
+            Ok(vec![Node::Const {
+                name: "a".into(),
+                value: Some(123),
+                string: None
+            }])
+        );
+        assert_eq!(
+            string_to_ast("const a = 0xaa;"),
+            Ok(vec![Node::Const {
+                name: "a".into(),
+                value: Some(0xaa),
+                string: None
+            }])
+        );
+        assert_eq!(
+            string_to_ast("const a = -7;"),
+            Ok(vec![Node::Const {
+                name: "a".into(),
+                value: Some(-7),
+                string: None
+            }])
+        );
+        assert_eq!(
+            string_to_ast("const a = \"foo bar\";"),
+            Ok(vec![Node::Const {
+                name: "a".into(),
+                value: None,
+                string: Some("foo bar".into())
+            }])
+        )
+    }
+
+    #[test]
+    fn parse_structs() {
+        assert_eq!(
+            string_to_ast("struct Point { x, y }"),
+            Ok(vec![Node::Struct {
+                name: "Point".into(),
+                members: vec![
+                    Member {
+                        name: "x".into(),
+                        typename: None,
+                        size: None
+                    },
+                    Member {
+                        name: "y".into(),
+                        typename: None,
+                        size: None
+                    },
+                ]
+            }])
+        );
+
+        assert_eq!(
+            string_to_ast("struct Foo { bar[100] }"),
+            Ok(vec![Node::Struct {
+                name: "Foo".into(),
+                members: vec![Member {
+                    name: "bar".into(),
+                    typename: None,
+                    size: Some(100)
+                },]
+            }])
+        );
+
+        assert_eq!(
+            string_to_ast("struct Foo { bar:Thing[100] }"),
+            Ok(vec![Node::Struct {
+                name: "Foo".into(),
+                members: vec![Member {
+                    name: "bar".into(),
+                    typename: Some("Thing".into()),
+                    size: Some(100)
+                },]
+            }])
+        );
+    }
+}
@@ -0,0 +1,5 @@
+extern crate pest;
+#[macro_use]
+extern crate pest_derive;
+
+mod forge_parser;
@@ -0,0 +1,66 @@
+ // header comment (the initial space is purposeful)
+
+fn blah() {
+  // comment
+  foo = 3; // normal assignment
+  x = 3 * -2; // unary minus
+  y = 3 - -3;
+  blah = foo + 5 * bar / 2; // Complex expr in assignment
+  arr[3] = 7; // Subscript in lvalue
+  arr[n * 2 - 1] = foo + 7; // expr in subscript
+  a[3] = b[70 - foo]; // subscripted rvalues
+  b = foo * (bar + 3); // expr parens
+  x1 = something(); // fn calls
+  something(1); // unary calls, call statements
+  something(1, foo, bar + 7, ha[0]); // exprs in calls
+  s = "hello"; // strings
+  s2 = "hello\n\tthere\0";
+  print("bah");
+  f = &x;
+}
+
+// fn with args
+fn foo(a, b) {}
+
+// fn with an annotation
+fn foo2<org=0x400>(x) { return x * x; }
+
+fn iftest() {
+  if (blah) {
+    foo();
+  } else {
+    bar();
+  }
+}
+
+fn whiletest() {
+  while (foo) {
+    bar();
+  }
+}
+
+fn repeatTest(x) {
+  repeat(x * 3) a {
+    blah(whatever + a);
+  }
+}
+
+global foo;
+global bar:Thing;
+
+fn varTest(t:Some) {
+  var x;
+  var y = 2 + x;
+  var z:Thingy;
+}
+
+struct Foo {}
+struct Bar {
+  x, y, name:Str
+}
+struct Str {
+  chars[100]
+}
+const LEN = 100;
+const STR = "Hey";
+struct What { foo[100] }