A big refactor, step 1

forge_core/src/compiler.rs

@@ -3,7 +3,7 @@ use crate::ast::*;
 use std::collections::btree_map::Entry::Vacant;
 use std::collections::{BTreeMap};
 use std::fmt::{Display, Formatter};
-use crate::forge_parser::{ParseError, parse};
+use crate::parser::{ParseError, parse};
 
 #[derive(Eq, Clone, PartialEq, Debug)]
 pub struct CompileError(pub usize, pub usize, pub String);
@@ -908,7 +908,7 @@ pub fn build_boot(src: &str) -> Result<Vec<String>, CompileError> {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::forge_parser::parse;
+    use crate::parser::parse;
 
     #[test]
     fn test_eval_const() {

forge_core/src/forge.pest

@@ -1,96 +0,0 @@
-WHITESPACE = _{ " " | "\t" | NEWLINE }
-COMMENT = _{ "//" ~ (!"\n" ~ ANY)* }
-
-name_char = { ASCII_ALPHA_LOWER | ASCII_ALPHA_UPPER | "_" | "$" }
-name = @{ name_char ~ (name_char | ASCII_DIGIT)* }
-
-dec_number = @{ ASCII_NONZERO_DIGIT ~ ASCII_DIGIT* }
-hex_number = ${ "0x" ~ ASCII_HEX_DIGIT+ }
-bin_number = ${ "0b" ~ ASCII_BIN_DIGIT+ }
-oct_number = ${ "0o" ~ ASCII_OCT_DIGIT+ }
-dec_zero = @{ "0" }
-number = { dec_number | hex_number | bin_number | oct_number | dec_zero }
-
-escape = @{ "\\" ~ ("t" | "r" | "n" | "0" | "\\" | "\"") }
-string_inner = ${ !("\"" | "\\") ~ ANY | escape }
-string = ${ "\"" ~ string_inner* ~ "\"" }
-
-assignment = { lvalue ~ "=" ~ rvalue }
-lvalue = { arrayref | name }
-rvalue = { expr | string }
-
-add = { "+" }
-sub = { "-" }
-mul = { "*" }
-div = { "/" }
-modulus = { "%" }
-log_and = { "&&" }
-log_or = { "||" }
-bit_and = { "&" }
-bit_or = { "|" }
-xor = { "^" }
-lt = { "<" }
-le = { "<=" }
-gt = { ">" }
-ge = { ">=" }
-eq = { "==" }
-ne = { "!=" }
-lshift = { "<<" }
-rshift = { ">>" }
-prefix = { "-" | "!" }
-
-operator = _{
- add | sub |
- mul | div | modulus |
- log_and | log_or |
- bit_and | bit_or | xor |
- lshift | rshift |
- lt | le | gt | ge |
- eq | ne }
-expr = { prefix? ~ val ~ (operator ~ prefix? ~ val)* }
-val = { // These are what get evaluated and left on the stack:
-  number | // Literal numbers
-  ("(" ~ expr ~ ")") | // Nested parenthesized exprs
-  // Bare literal ids, fn calls, array references
-  // (call, or pointer arithmetic, since ids all reference addresses)
-  // (multiple calls / subscripts aren't allowed; use CALL or collapse to one
-  // subscript)
-  call | arrayref | name | address }
-
-address = { "&" ~ name } // The address for a name
-arrayref = { name ~ subscript } // Subscripting is only literal IDs. Anything else can be pointer arithmetic
-call = { name ~ arglist } // Call syntax is only literal IDs. Calling things dynamically is done with a builtin; CALL(foo[3], 1, 2, 3)
-arglist = { "(" ~ (rvalue ~ ("," ~ rvalue)*)? ~ ")" }
-subscript = { "[" ~ expr ~ "]" }
-
-statement = { ((return_stmt | assignment | call | var_decl) ~ ";") | conditional | while_loop | repeat_loop }
-
-block = { "{" ~ statement* ~ "}" }
-
-function = { "fn" ~ name ~ annotations? ~ argnames ~ block }
-annotations = { "<" ~ (annotation ~ ("," ~ annotation)*) ~ ">" }
-annotation = { inline_annotation | org_annotation | type_annotation }
-inline_annotation = { "inline" }
-org_annotation = { "org=" ~ number }
-type_annotation = { "type=" ~ name }
-
-argnames = { "(" ~ (argname ~ ("," ~ argname)*)? ~ ")" }
-argname = { name ~ typename? }
-
-declaration = { function | global | struct_decl | const_decl }
-program = { (COMMENT | WHITESPACE)? ~ declaration* ~ EOI }
-
-return_stmt = { "return" ~ expr? }
-conditional = { "if" ~ "(" ~ expr ~ ")" ~ block ~ ("else" ~ block)? }
-while_loop = { "while" ~ "(" ~ expr ~ ")" ~ block }
-repeat_loop = { "repeat" ~ "(" ~ expr ~ ")" ~ name? ~ block }
-var_decl = { "var" ~ name ~ varinfo ~ ("=" ~ expr)? }
-global = { "global" ~ name ~ varinfo ~ ";" }
-typename = { ":" ~ name }
-size = { "[" ~ expr ~ "]" }
-const_decl = { "const" ~ name ~ "=" ~ (expr | string) ~ ";" }
-
-struct_decl = { "struct" ~ name ~ "{" ~ members ~ "}" }
-member = { name ~ varinfo }
-members = { (member ~ ("," ~ member)*)? }
-varinfo = { typename? ~ size? }

forge_core/src/lib.rs

@@ -4,4 +4,4 @@ extern crate pest_derive;
 
 mod ast;
 pub mod compiler;
-mod forge_parser;
+mod parser;

forge_core/src/parser/mod.rs

@@ -2,7 +2,7 @@ use pest::pratt_parser::PrattParser;
 use pest::Parser;
 
 #[derive(Parser)]
-#[grammar = "new.pest"]
+#[grammar = "parser/forge.pest"]
 struct ForgeParser;
 
 lazy_static::lazy_static! {
@@ -28,8 +28,10 @@ lazy_static::lazy_static! {
     };
 }
 
+/// Expose Pest's generated `Rule` type so other things can see it.
+pub type PestRule = Rule;
+
 use crate::ast::*;
-use pest::error::{Error, LineColLocation};
 use std::iter::Peekable;
 use std::str::FromStr;
 
@@ -109,19 +111,10 @@ impl PairsExt for Peekable<Pairs<'_>> {
     }
 }
 
-#[derive(Eq, PartialEq, Clone, Debug)]
-pub struct ParseError(pub usize, pub usize, pub String);
+mod parse_error;
+mod pairs_ext;
 
-impl From<pest::error::Error<Rule>> for ParseError {
-    fn from(err: Error<Rule>) -> Self {
-        let message = err.to_string();
-        match err.line_col {
-            LineColLocation::Pos((line, col)) | LineColLocation::Span((line, col), _) => {
-                Self(line, col, message)
-            }
-        }
-    }
-}
+pub use parse_error::ParseError;
 
 trait AstNode: Sized {
     const RULE: Rule;

forge_core/src/parser/pairs_ext.rs

@@ -0,0 +1,91 @@
+use std::iter::Peekable;
+use std::str::FromStr;
+use crate::parser::{Pair, Pairs, PestRule};
+
+/// Some handy utility methods to extend Pair with
+trait PairExt {
+    /// Go down one level in the AST, return the first node. Assumes the first node exists
+    fn first(self) -> Self;
+
+    /// Returns the first child node as a String; useful for things like names and numbers
+    /// which are really just typed strings
+    fn first_as_string(self) -> String;
+
+    /// Just like `first` but panics if there's more than one child node.
+    fn only(self) -> Self;
+
+    /// This should really be an AstNode or something, but it's used so often: turn a Pair into
+    /// an i32 by parsing the Forge number format (0xwhatever, 0bwhatever, etc)
+    fn into_number(self) -> i32;
+
+    /// Create a String containing the string represented by a given pair. Since the pair will
+    /// reference bytes containing escape sequences, this isn't the same as an &str to the
+    /// original code; this is a new string translating those escape sequences to their actual
+    /// bytes.
+    fn into_quoted_string(self) -> String;
+}
+
+impl<'a> PairExt for Pair<'a> {
+    fn first(self) -> Self {
+        self.into_inner().next().unwrap()
+    }
+
+    fn first_as_string(self) -> String {
+        String::from(self.first().as_str())
+    }
+
+    fn only(self) -> Pair<'a> {
+        let mut iter = self.into_inner();
+        let child = iter.next().unwrap();
+        debug_assert_eq!(iter.next(), None);
+        child
+    }
+
+    fn into_number(self) -> i32 {
+        let first = self.into_inner().next().unwrap();
+        match first.as_rule() {
+            PestRule::dec_number | PestRule::dec_zero => i32::from_str(first.as_str()).unwrap(),
+            PestRule::hex_number => i32::from_str_radix(first.as_str().get(2..).unwrap(), 16).unwrap(),
+            PestRule::bin_number => i32::from_str_radix(first.as_str().get(2..).unwrap(), 2).unwrap(),
+            PestRule::oct_number => i32::from_str_radix(first.as_str().get(2..).unwrap(), 8).unwrap(),
+            _ => panic!("Expected a number, got {}", first.as_str()),
+        }
+    }
+
+    fn into_quoted_string(self) -> String {
+        let mut string = String::with_capacity(self.as_str().len());
+        for inner in self.into_inner() {
+            let string_inner = inner.as_str();
+            match string_inner {
+                "\\t" => string.push('\t'),
+                "\\r" => string.push('\r'),
+                "\\n" => string.push('\n'),
+                "\\0" => string.push('\0'),
+                "\\\\" => string.push('\\'),
+                "\\\"" => string.push('\"'),
+                _ => string.push_str(string_inner),
+            }
+        }
+        string
+    }
+}
+
+/// Some handy utility methods to extend Pairs with
+trait PairsExt {
+    /// Peek the next sibling but only if it matches a given rule: used for things
+    /// with optional modifiers following
+    fn next_if_rule(&mut self, rule: PestRule) -> Option<Pair>;
+
+    /// Grab the first thing from the list if you're sure it exists
+    fn first(&mut self) -> Pair;
+}
+
+impl PairsExt for Peekable<Pairs<'_>> {
+    fn next_if_rule(&mut self, rule: PestRule) -> Option<Pair> {
+        self.next_if(|p| p.as_rule() == rule)
+    }
+
+    fn first(&mut self) -> Pair {
+        self.next().unwrap().first()
+    }
+}

forge_core/src/parser/parse_error.rs

@@ -0,0 +1,16 @@
+use pest::error::{Error, LineColLocation};
+use crate::parser::PestRule;
+
+#[derive(Eq, PartialEq, Clone, Debug)]
+pub struct ParseError(pub usize, pub usize, pub String);
+
+impl From<Error<PestRule>> for ParseError {
+    fn from(err: Error<PestRule>) -> Self {
+        let message = err.to_string();
+        match err.line_col {
+            LineColLocation::Pos((line, col)) | LineColLocation::Span((line, col), _) => {
+                Self(line, col, message)
+            }
+        }
+    }
+}