Working on parser. It can parse return statements and log errors now

src/lexer/mod.rs

@@ -18,7 +18,7 @@ lazy_static! {
     };
 }
 
-#[derive(Debug, PartialEq, Eq, Clone, Hash)]
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Hash)]
 pub enum TokenType {
     Illegal,
     EOF,

src/parser/ast/mod.rs

@@ -3,7 +3,7 @@ use crate::{
     parser::Parser,
 };
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub struct Program {
     pub statements: Vec<Statement>,
 }
@@ -16,13 +16,15 @@ pub enum Node {
 #[derive(Debug, PartialEq)]
 pub enum Statement {
     Let(LetStatement),
+    Return(ReturnStatement),
 }
 
 impl<'a> Statement {
     pub fn parse(parser: &'a mut Parser, token: Token) -> Option<Self> {
         match token.name {
             TokenType::Let => Some(Statement::Let(LetStatement::parse(parser)?)),
-            _ => todo!(),
+            TokenType::Return => Some(Statement::Return(ReturnStatement::parse(parser)?)),
+            _ => None,
         }
     }
 }
@@ -57,12 +59,28 @@ impl LetStatement {
 
         // TODO: Right now, We are just skipping over all the expressions
         // That'll come later
+        // Also, Right now, It hangs forever in case there is no semicolon at the end
         while parser.lexer.next() != Some(Token::new(TokenType::Semicolon)) {}
 
         Some(stmt)
     }
 }
 
+#[derive(Debug, PartialEq)]
+pub struct ReturnStatement {
+    return_value: Option<Expression>,
+}
+
+impl ReturnStatement {
+    fn parse(parser: &mut Parser) -> Option<Self> {
+        let stmt = ReturnStatement {
+            return_value: Some(Expression),
+        };
+        while parser.lexer.next() != Some(Token::new(TokenType::Semicolon)) {}
+        return Some(stmt);
+    }
+}
+
 // Identifier will be an expression
 // Identifier in a let statement like, let x = 5; where `x` is an identifier doesn't produce a value
 // but an identifier *can* produce value when used on rhs, e.g. let x = y; Here `y` is producing a value

src/parser/mod.rs

@@ -4,32 +4,35 @@ use {
         lexer::{Lexer, Token, TokenType},
         parser::ast::{Program, Statement},
     },
-    std::iter::Peekable,
+    std::{
+        fmt::{Display, Error as FmtError, Formatter},
+        iter::Peekable,
+    },
 };
 
 pub struct Parser<'a> {
     lexer: Peekable<Lexer<'a>>,
+    pub errors: Vec<ParserError>,
 }
 
 impl<'a> Parser<'a> {
     pub fn new(lexer: Lexer<'a>) -> Self {
         Self {
             lexer: lexer.peekable(),
+            errors: vec![],
         }
     }
 
-    pub fn parse_program(mut self) -> Program {
+    pub fn parse_program(&mut self) -> Program {
         let mut program = Program { statements: vec![] };
 
-        loop {
+        while let Some(token) = self.lexer.next() {
-            let token = self.lexer.next().unwrap();
             if token.name == TokenType::EOF {
                 break;
             }
-
+            match Statement::parse(self, token.clone()) {
-            match Statement::parse(&mut self, token) {
                 Some(v) => program.statements.push(v),
-                None => todo!(), // This will happen in case of a parsing error or something
+                None => {} // This will happen in case of a parsing error or something
             }
         }
 
@@ -52,9 +55,32 @@ impl<'a> Parser<'a> {
         if self.peek_token_is(token) {
             self.lexer.next()
         } else {
+            let got_token = match self.lexer.peek() {
+                Some(v) => Some(v.name),
+                None => None,
+            };
+            self.peek_error(token, got_token);
             None
         }
     }
+
+    fn peek_error(&mut self, et: TokenType, gt: Option<TokenType>) {
+        let msg = match gt {
+            Some(v) => format!("expected next token to be {:?}, Got {:?} instead", et, v),
+            None => format!("expected next token to be {:?}, Got None instead", et),
+        };
+        self.errors.push(ParserError { reason: msg });
+    }
+}
+
+pub struct ParserError {
+    reason: String,
+}
+
+impl Display for ParserError {
+    fn fmt(&self, fmt: &mut Formatter) -> Result<(), FmtError> {
+        write!(fmt, "{}", self.reason)
+    }
 }
 
 #[cfg(test)]
@@ -62,34 +88,68 @@ mod tests {
     use crate::{
         lexer::{Lexer, TokenType},
         parser::{
-            ast::{Identifier, LetStatement, Statement},
+            ast::{Identifier, LetStatement, Program, Statement},
             Parser,
         },
     };
     #[test]
     fn let_statements() {
-        let lexer = Lexer::new("let x =5;let y=10; let foobar=538383;");
+        let mut lexer = Lexer::new("let x =5;let y=10; let foobar=538383;");
-        let parser = Parser::new(lexer);
+        let mut parser = Parser::new(lexer);
+        let program = parser.parse_program();
+        check_parser_errors(&parser);
+        assert_eq!(program.statements.len(), 3);
+        assert_eq!(
+            program,
+            Program {
+                statements: vec![
+                    Statement::Let(LetStatement {
+                        name: Identifier::new(TokenType::Let, "x"),
+                        value: None
+                    }),
+                    Statement::Let(LetStatement {
+                        name: Identifier::new(TokenType::Let, "y"),
+                        value: None
+                    }),
+                    Statement::Let(LetStatement {
+                        name: Identifier::new(TokenType::Let, "foobar"),
+                        value: None
+                    })
+                ],
+            }
+        );
+
+        lexer = Lexer::new("let x = 5;let x 5;let = 10; let 83838383;");
+        parser = Parser::new(lexer);
+        let program = parser.parse_program();
+        check_parser_errors(&parser);
+        assert_eq!(parser.errors.len(), 3);
+        assert_eq!(program.statements.len(), 1);
+    }
+
+    #[test]
+    fn return_statements() {
+        let lexer = Lexer::new("return 5; return 10; return add(10);");
+        let mut parser = Parser::new(lexer);
         let program = parser.parse_program();
 
+        check_parser_errors(&parser);
         assert_eq!(program.statements.len(), 3);
+        assert_eq!(parser.errors.len(), 0);
+    }
 
-        assert_eq!(
+    fn check_parser_errors(p: &Parser) {
-            program.statements,
+        if p.errors.is_empty() {
-            vec![
+            return;
-                Statement::Let(LetStatement {
+        } else {
-                    name: Identifier::new(TokenType::Let, "x"),
+            let mut out = String::new();
-                    value: None
+
-                }),
+            out.push_str(&format!("parser has {} errors\n", p.errors.len()));
-                Statement::Let(LetStatement {
+
-                    name: Identifier::new(TokenType::Let, "y"),
+            for error in &p.errors {
-                    value: None
+                out.push_str(&format!("parser error: {}\n", error));
-                }),
+            }
-                Statement::Let(LetStatement {
+            eprintln!("{}", out);
-                    name: Identifier::new(TokenType::Let, "foobar"),
+        }
-                    value: None
-                })
-            ]
-        );
     }
 }

src/repl.rs

@@ -24,7 +24,7 @@ fn start<R: BufRead, W: Write>(mut ip: R, mut out: W) {
             println!("{:?}", token);
         }
 
-        let parser = Parser::new(tokens);
+        let mut parser = Parser::new(tokens);
 
         let stmts = parser.parse_program();
         println!("parser={:?}", stmts);