Refactoring the parser some

forge_core/src/ast.rs

@@ -0,0 +1 @@
+use crate::forge_parser::Pair;

forge_core/src/forge.pest

@@ -12,6 +12,10 @@ oct_number = ${ "0o" ~ ASCII_OCT_DIGIT+ }
 dec_zero = @{ "0" }
 number = { dec_number | hex_number | bin_number | oct_number | dec_zero | neg_number }
 
+escape = @{ "\\" ~ ("t" | "r" | "n" | "0" | "\\" | "\"") }
+string_inner = ${ !("\"" | "\\") ~ ANY | escape }
+string = ${ "\"" ~ string_inner* ~ "\"" }
+
 assignment = { lvalue ~ "=" ~ rvalue }
 lvalue = { arrayref | name }
 rvalue = { expr | string }
@@ -37,11 +41,7 @@ arg = { expr | string }
 arglist = { "(" ~ (arg ~ ("," ~ arg)*)? ~ ")" }
 subscript = { "[" ~ expr ~ "]" }
 
-escape = @{ "\\" ~ ("t" | "r" | "n" | "0" | "\\" | "\"") }
+statement = { ((return_stmt | assignment | call | var_decl) ~ ";") | conditional | loop_stmt }
-string_inner = ${ !("\"" | "\\") ~ ANY | escape }
-string = ${ "\"" ~ string_inner* ~ "\"" }
-
-statement = { ((return_stmt | assignment | call | declaration) ~ ";") | conditional | loop_stmt }
 
 block = { "{" ~ statement* ~ "}" }
 
@@ -50,18 +50,21 @@ annotations = { "<" ~ (annotation ~ ("," ~ annotation)*) ~ ">" }
 annotation = { "inline" | ("org=" ~ number) | ("type=" ~ name) }
 argnames = { "(" ~ (name ~ typename? ~ ("," ~ name ~ typename?)*)? ~ ")" }
 
-program = { (COMMENT | WHITESPACE)? ~ (function | global | struct_decl | const_decl)* ~ EOI }
+declaration = { function | global | struct_decl | const_decl }
+program = { (COMMENT | WHITESPACE)? ~ declaration* ~ EOI }
 
 return_stmt = { "return" ~ expr }
 conditional = { "if" ~ "(" ~ expr ~ ")" ~ block ~ ("else" ~ block)? }
 loop_stmt = { while_stmt | repeat }
 while_stmt = { "while" ~ "(" ~ expr ~ ")" ~ block }
 repeat = { "repeat" ~ "(" ~ expr ~ ")" ~ name ~ block }
-declaration = { "var" ~ name ~ typename? ~ ("=" ~ expr)? }
+var_decl = { "var" ~ name ~ varinfo ~ ("=" ~ expr)? }
-global = { "global" ~ name ~ typename? ~ ";" }
+global = { "global" ~ name ~ varinfo ~ ";" }
 typename = { ":" ~ name }
+size = { "[" ~ number ~ "]" }
 const_decl = { "const" ~ name ~ "=" ~ (number | string) ~ ";" }
 
 struct_decl = { "struct" ~ name ~ "{" ~ members ~ "}" }
-member = { name ~ typename? ~ ("[" ~ number ~ "]")? }
+member = { name ~ varinfo }
 members = { (member ~ ("," ~ member)*)? }
+varinfo = { typename? ~ size? }

forge_core/src/forge_parser.rs

@@ -1,4 +1,4 @@
-use pest::Parser;
+use pest::{Parser, RuleType};
 
 mod inner {
     #[derive(Parser)]
@@ -8,11 +8,41 @@ mod inner {
 
 use inner::*;
 use pest::error::{Error, LineColLocation};
+use std::iter::Peekable;
 use std::str::FromStr;
 
-type Pair<'a> = pest::iterators::Pair<'a, Rule>;
+pub(crate) type Pair<'a> = pest::iterators::Pair<'a, Rule>;
+pub(crate) type Pairs<'i, R = Rule> = pest::iterators::Pairs<'i, R>;
 
-fn parse_i32(pair: Pair) -> i32 {
+trait Children {
+    fn first(self) -> Self;
+    fn only(self) -> Self;
+}
+
+impl<'a> Children for Pair<'a> {
+    fn first(self) -> Self {
+        self.into_inner().next().unwrap()
+    }
+
+    fn only(self) -> Pair<'a> {
+        let mut iter = self.into_inner();
+        let child = iter.next().unwrap();
+        debug_assert_eq!(iter.next(), None);
+        child
+    }
+}
+
+trait PairsExt {
+    fn next_if_rule(&mut self, rule: Rule) -> Option<Pair>;
+}
+
+impl PairsExt for Peekable<Pairs<'_>> {
+    fn next_if_rule(&mut self, rule: Rule) -> Option<Pair> {
+        self.next_if(|p| p.as_rule() == rule)
+    }
+}
+
+fn parse_number(pair: Pair) -> i32 {
     let first = pair.into_inner().next().unwrap();
     match first.as_rule() {
         Rule::dec_number | Rule::dec_zero | Rule::neg_number => {
@@ -29,7 +59,7 @@ fn parse_i32(pair: Pair) -> i32 {
 /// reference bytes containing escape sequences, this isn't the same as an &str to the
 /// original code; this is a new string translating those escape sequences to their actual
 /// bytes.
-fn create_string(pair: Pair) -> String {
+fn parse_string(pair: Pair) -> String {
     let mut string = String::with_capacity(pair.as_str().len());
     for inner in pair.into_inner() {
         let string_inner = inner.as_str();
@@ -46,6 +76,10 @@ fn create_string(pair: Pair) -> String {
     string
 }
 
+fn parse_name(pair: Pair) -> &str {
+    pair.as_str()
+}
+
 #[derive(Eq, PartialEq, Clone, Debug)]
 pub struct ParseError(usize, usize, String);
 
@@ -60,27 +94,111 @@ impl From<pest::error::Error<Rule>> for ParseError {
     }
 }
 
+trait Parseable<'a>: From<Pair<'a>> {
+    const RULE: Rule;
+    fn parse(src: &'a str) -> Result<Self, ParseError> {
+        let pair = ForgeParser::parse(Self::RULE, src)
+            .map_err(ParseError::from)?
+            .next()
+            .unwrap();
+        Ok(pair.into())
+    }
+}
+
 #[derive(Eq, PartialEq, Clone, Debug)]
-pub enum Node {
+pub enum Declaration {
-    Function {
+    Function(Function),
-        name: String,
+    Global(Global),
-        org: Option<i32>,
+    Struct(Struct),
-        typename: Option<String>,
+    Const(Const),
-        inline: bool,
+}
-    },
+
-    Global {
+impl<'a> From<Pair<'a>> for Declaration {
-        name: String,
+    fn from(pair: Pair) -> Self {
-        typename: Option<String>,
+        match pair.as_rule() {
-    },
+            Rule::function => todo!(),
-    Struct {
+            Rule::global => Self::Global(pair.into()),
-        name: String,
+            Rule::struct_decl => Self::Struct(pair.into()),
-        members: Vec<Member>,
+            Rule::const_decl => Self::Const(pair.into()),
-    },
+            _ => todo!(),
-    Const {
+        }
-        name: String,
+    }
-        value: Option<i32>,
+}
-        string: Option<String>,
+
-    },
+impl Parseable<'_> for Declaration {
+    const RULE: Rule = Rule::declaration;
+}
+
+#[derive(Eq, PartialEq, Clone, Debug, Default)]
+pub struct Varinfo {
+    typename: Option<String>,
+    size: Option<i32>,
+}
+
+impl Varinfo {
+    fn read_or_default(mut pairs: Peekable<Pairs>) -> Self {
+        pairs
+            .next_if_rule(Rule::varinfo)
+            .map_or(Self::default(), |v| v.into())
+    }
+}
+
+impl From<Pair<'_>> for Varinfo {
+    fn from(pair: Pair<'_>) -> Self {
+        let mut children = pair.into_inner().peekable();
+        let typename = children
+            .next_if_rule(Rule::typename)
+            .map(|t| String::from(t.first().as_str()));
+        let size = children
+            .next_if_rule(Rule::size)
+            .map(|s| parse_number(s.first()));
+        Self { typename, size }
+    }
+}
+
+#[derive(Eq, PartialEq, Clone, Debug)]
+pub struct Global {
+    name: String,
+    typename: Option<String>,
+    size: Option<i32>,
+}
+
+impl From<Pair<'_>> for Global {
+    fn from(global: Pair) -> Self {
+        let mut inner = global.into_inner().peekable();
+        let name = String::from(inner.next().unwrap().as_str());
+        let varinfo = Varinfo::read_or_default(inner);
+
+        Global {
+            name,
+            typename: varinfo.typename,
+            size: varinfo.size,
+        }
+    }
+}
+
+impl Parseable<'_> for Global {
+    const RULE: Rule = Rule::global;
+}
+
+#[derive(Eq, PartialEq, Clone, Debug)]
+pub struct Struct {
+    name: String,
+    members: Vec<Member>,
+}
+
+impl From<Pair<'_>> for Struct {
+    fn from(pair: Pair<'_>) -> Self {
+        let mut inner = pair.into_inner();
+        let name = String::from(inner.next().unwrap().as_str());
+        let member_pairs = inner.next().unwrap().into_inner();
+        let members: Vec<_> = member_pairs.map(|member| member.into()).collect();
+        Struct { name, members }
+    }
+}
+
+impl Parseable<'_> for Struct {
+    const RULE: Rule = Rule::struct_decl;
 }
 
 #[derive(Eq, PartialEq, Clone, Debug)]
@@ -90,82 +208,62 @@ pub struct Member {
     size: Option<i32>,
 }
 
-pub fn string_to_ast(string: &str) -> Result<Vec<Node>, ParseError> {
+impl From<Pair<'_>> for Member {
-    let pairs = ForgeParser::parse(Rule::program, string)
+    fn from(pair: Pair<'_>) -> Self {
-        .map_err(ParseError::from)?
+        let mut inner = pair.into_inner().peekable();
-        .next()
+        let name = String::from(inner.next().unwrap().as_str());
-        .unwrap()
+        let varinfo = Varinfo::read_or_default(inner);
-        .into_inner();
 
-    let mut decls = Vec::new();
+        Member {
-    for pair in pairs {
+            name,
-        let r = pair.as_rule();
+            typename: varinfo.typename,
-        match r {
+            size: varinfo.size,
-            Rule::function => {}
+        }
-            Rule::global => decls.push(parse_global(pair)),
+    }
-            Rule::struct_decl => decls.push(parse_struct(pair)),
+}
-            Rule::const_decl => decls.push(parse_const(pair)),
+
-            Rule::EOI => {} // Ignore EOI; we have to capture it but it doesn't do anything
+impl Parseable<'_> for Member {
+    const RULE: Rule = Rule::member;
+}
+
+#[derive(Eq, PartialEq, Clone, Debug)]
+pub struct Const {
+    name: String,
+    value: Option<i32>,
+    string: Option<String>,
+}
+
+impl From<Pair<'_>> for Const {
+    fn from(pair: Pair) -> Self {
+        let mut inner = pair.into_inner();
+        let name = String::from(inner.next().unwrap().as_str());
+        let value = inner.next().unwrap();
+        match value.as_rule() {
+            Rule::string => Const {
+                name,
+                string: Some(String::from(parse_string(value))),
+                value: None,
+            },
+            Rule::number => Const {
+                name,
+                string: None,
+                value: Some(parse_number(value)),
+            },
             _ => unreachable!(),
         }
     }
-
-    Ok(decls)
 }
 
-fn parse_global(global: Pair) -> Node {
+impl Parseable<'_> for Const {
-    let mut inner = global.into_inner();
+    const RULE: Rule = Rule::const_decl;
-    let name = String::from(inner.next().unwrap().as_str());
-    let typename = inner
-        .next()
-        .map(|p| String::from(p.into_inner().next().unwrap().as_str()));
-    Node::Global { name, typename }
 }
 
-fn parse_const(pair: Pair) -> Node {
+#[derive(Eq, PartialEq, Clone, Debug)]
-    let mut inner = pair.into_inner();
+pub struct Function {
-    let name = String::from(inner.next().unwrap().as_str());
+    name: String,
-    let value = inner.next().unwrap();
+    org: Option<i32>,
-    match value.as_rule() {
+    typename: Option<String>,
-        Rule::string => Node::Const {
+    inline: bool,
-            name,
-            string: Some(String::from(create_string(value))),
-            value: None,
-        },
-        Rule::number => Node::Const {
-            name,
-            string: None,
-            value: Some(parse_i32(value)),
-        },
-        _ => unreachable!(),
-    }
-}
-
-fn parse_member(pair: Pair) -> Member {
-    let mut inner = pair.into_inner();
-    let mut member = Member {
-        name: String::from(inner.next().unwrap().as_str()),
-        typename: None,
-        size: None,
-    };
-    for child in inner {
-        match child.as_rule() {
-            Rule::number => member.size = Some(parse_i32(child)),
-            Rule::typename => {
-                member.typename = Some(String::from(child.into_inner().next().unwrap().as_str()))
-            }
-            _ => unreachable!(),
-        }
-    }
-    member
-}
-
-fn parse_struct(pair: Pair) -> Node {
-    let mut inner = pair.into_inner();
-    let name = String::from(inner.next().unwrap().as_str());
-    let mut member_pairs = inner.next().unwrap().into_inner();
-    let members: Vec<_> = member_pairs.map(|member| parse_member(member)).collect();
-    Node::Struct { name, members }
 }
 
 #[cfg(test)]
@@ -174,66 +272,81 @@ mod test {
 
     #[test]
     fn parse_globals() {
-        let ast = string_to_ast("global foo;");
         assert_eq!(
-            ast,
+            Global::parse("global foo;"),
-            Ok(vec![Node::Global {
+            Ok(Global {
                 name: "foo".into(),
-                typename: None
+                typename: None,
-            }])
+                size: None
+            })
         );
-
-        let ast = string_to_ast("global bar:Sometype;");
         assert_eq!(
-            ast,
+            Global::parse("global foo:Thing;"),
-            Ok(vec![Node::Global {
+            Ok(Global {
-                name: "bar".into(),
+                name: "foo".into(),
-                typename: Some("Sometype".into())
+                typename: Some("Thing".into()),
-            }])
+                size: None
+            })
+        );
+        assert_eq!(
+            Global::parse("global foo:Thing[10];"),
+            Ok(Global {
+                name: "foo".into(),
+                typename: Some("Thing".into()),
+                size: Some(10)
+            })
+        );
+        assert_eq!(
+            Global::parse("global foo[10];"),
+            Ok(Global {
+                name: "foo".into(),
+                typename: None,
+                size: Some(10)
+            })
         );
     }
 
     #[test]
     fn parse_consts() {
         assert_eq!(
-            string_to_ast("const a = 123;"),
+            Const::parse("const a = 123;"),
-            Ok(vec![Node::Const {
+            Ok(Const {
                 name: "a".into(),
                 value: Some(123),
                 string: None
-            }])
+            })
         );
         assert_eq!(
-            string_to_ast("const a = 0xaa;"),
+            Const::parse("const a = 0xaa;"),
-            Ok(vec![Node::Const {
+            Ok(Const {
                 name: "a".into(),
                 value: Some(0xaa),
                 string: None
-            }])
+            })
         );
         assert_eq!(
-            string_to_ast("const a = -7;"),
+            Const::parse("const a = -7;"),
-            Ok(vec![Node::Const {
+            Ok(Const {
                 name: "a".into(),
                 value: Some(-7),
                 string: None
-            }])
+            })
         );
         assert_eq!(
-            string_to_ast("const a = \"foo bar\";"),
+            Const::parse("const a = \"foo bar\";"),
-            Ok(vec![Node::Const {
+            Ok(Const {
                 name: "a".into(),
                 value: None,
                 string: Some("foo bar".into())
-            }])
+            })
         )
     }
 
     #[test]
     fn parse_structs() {
         assert_eq!(
-            string_to_ast("struct Point { x, y }"),
+            Struct::parse("struct Point { x, y }"),
-            Ok(vec![Node::Struct {
+            Ok(Struct {
                 name: "Point".into(),
                 members: vec![
                     Member {
@@ -247,31 +360,31 @@ mod test {
                         size: None
                     },
                 ]
-            }])
+            })
         );
 
         assert_eq!(
-            string_to_ast("struct Foo { bar[100] }"),
+            Struct::parse("struct Foo { bar[100] }"),
-            Ok(vec![Node::Struct {
+            Ok(Struct {
                 name: "Foo".into(),
                 members: vec![Member {
                     name: "bar".into(),
                     typename: None,
                     size: Some(100)
                 },]
-            }])
+            })
         );
 
         assert_eq!(
-            string_to_ast("struct Foo { bar:Thing[100] }"),
+            Struct::parse("struct Foo { bar:Thing[100] }"),
-            Ok(vec![Node::Struct {
+            Ok(Struct {
                 name: "Foo".into(),
                 members: vec![Member {
                     name: "bar".into(),
                     typename: Some("Thing".into()),
                     size: Some(100)
                 },]
-            }])
+            })
         );
     }
 }

forge_core/src/lib.rs

@@ -2,4 +2,5 @@ extern crate pest;
 #[macro_use]
 extern crate pest_derive;
 
+mod ast;
 mod forge_parser;