Completed Let/Return Statement parser

1. Completed Expression parsing in Let/Return Statement Parsers
2. Removed most of the TODOs
3. Added more tests

src/parser/ast/mod.rs

@@ -1,4 +1,3 @@
-// TODO: Maybe implement String method to pretty print all AST nodes
 use {
     crate::{
         lexer::{Token, TokenType},
@@ -71,26 +70,26 @@ impl LetStatement {
     pub fn new(name: Identifier) -> Self {
         Self { name, value: None }
     }
-    pub fn with_value(name: Identifier, value: Option<Expression>) -> Self {
+    pub fn with_value(name: Identifier, value: Expression) -> Self {
-        Self { name, value }
+        Self {
+            name,
+            value: Some(value),
+        }
     }
-    // TODO: Implement code to parse let statement
+
     pub fn parse(parser: &mut Parser) -> Option<Self> {
-        let mut stmt = LetStatement {
-            name: Identifier::new(TokenType::Let, "placeholder_value"),
-            value: None,
-        };
-
         let ident = parser.expect_peek(TokenType::Ident)?;
-        stmt.name.value = ident.literal?;
+
         parser.expect_peek(TokenType::Assign)?;
+        let ntoken = parser.lexer.next()?;
+        let expr = Expression::parse(parser, ntoken, ExpressionPriority::Lowest);
 
-        // TODO: Right now, We are just skipping over all the expressions
+        parser.expect_peek(TokenType::Semicolon);
-        // That'll come later
-        // Also, Right now, It hangs forever in case there is no semicolon at the end
-        while parser.lexer.next() != Some(Token::new(TokenType::Semicolon)) {}
 
-        Some(stmt)
+        Some(Self {
+            name: Identifier::new(TokenType::Let, &ident.literal?),
+            value: expr,
+        })
     }
 
     const fn token_literal() -> &'static str {
@@ -117,12 +116,17 @@ pub struct ReturnStatement {
 }
 
 impl ReturnStatement {
+    pub fn new(expr: Expression) -> Self {
+        ReturnStatement {
+            return_value: Some(expr),
+        }
+    }
+
     fn parse(parser: &mut Parser) -> Option<Self> {
-        let stmt = ReturnStatement {
+        let token = parser.lexer.next()?;
-            return_value: Some(Expression::None),
+        let expr = Expression::parse(parser, token, ExpressionPriority::Lowest);
-        };
+        parser.expect_peek(TokenType::Semicolon);
-        while parser.lexer.next() != Some(Token::new(TokenType::Semicolon)) {}
+        return Some(ReturnStatement { return_value: expr });
-        return Some(stmt);
     }
 
     const fn token_literal() -> &'static str {
@@ -193,8 +197,6 @@ pub enum Expression {
     IfExpression(IfExpression),
     FunctionExpression(FunctionLiteral),
     CallExpression(CallExpression),
-    // TODO: Temporary placeholder value. Should be removed once this section is done
-    None,
 }
 
 impl Expression {
@@ -249,7 +251,6 @@ impl Display for Expression {
             Expression::IfExpression(v) => v.to_string(),
             Expression::FunctionExpression(v) => v.to_string(),
             Expression::CallExpression(v) => v.to_string(),
-            Expression::None => "None".into(),
         };
 
         f.write_str(&value)

src/parser/mod.rs

@@ -15,7 +15,6 @@ type PrefixParseFn = fn(&mut Parser, token: Token) -> Option<Expression>;
 type InfixParseFn = fn(&mut Parser, Token, Expression) -> Option<Expression>;
 
 //TODO: Add Parser tracing so we can easily figure out the call stack hierarchy
-
 lazy_static! {
     static ref PRECEDENCE_MAP: HashMap<TokenType, ExpressionPriority> = {
         let mut m = HashMap::new();
@@ -186,34 +185,74 @@ mod tests {
 
     #[test]
     fn let_statements() {
-        let mut lexer = Lexer::new("let x =5;let y=10; let foobar=538383;");
+        let test_cases = [(
+            "let x =5;let y=10; let foobar=538383;",
+            vec![
+                Statement::Let(LetStatement::with_value(
+                    Identifier::new(TokenType::Let, "x"),
+                    Expression::IntegerLiteral(IntegerLiteral::new(5)),
+                )),
+                Statement::Let(LetStatement::with_value(
+                    Identifier::new(TokenType::Let, "y"),
+                    Expression::IntegerLiteral(IntegerLiteral::new(10)),
+                )),
+                Statement::Let(LetStatement::with_value(
+                    Identifier::new(TokenType::Let, "foobar"),
+                    Expression::IntegerLiteral(IntegerLiteral::new(538383)),
+                )),
+            ],
+        )];
+
+        for test in test_cases.iter() {
+            let lexer = Lexer::new(test.0);
+            let mut parser = Parser::new(lexer);
+            let program = parser.parse_program();
+            check_parser_errors(&parser);
+            assert_eq!(parser.errors.len(), 0);
+            assert!(program.is_some());
+            assert_eq!(program.unwrap().statements, test.1);
+        }
+
+        let fail_case = "let x 5; let 10; let 83838383;";
+
+        let lexer = Lexer::new(fail_case);
         let mut parser = Parser::new(lexer);
         let program = parser.parse_program();
         check_parser_errors(&parser);
-        assert_eq!(parser.errors.len(), 0);
-        assert!(program.is_some());
-        let program = program.unwrap();
-        assert_eq!(program.statements.len(), 3);
-        assert_eq!(
-            program,
-            Program {
-                statements: vec![
-                    Statement::Let(LetStatement::new(Identifier::new(TokenType::Let, "x"),)),
-                    Statement::Let(LetStatement::new(Identifier::new(TokenType::Let, "y"),)),
-                    Statement::Let(LetStatement::new(Identifier::new(TokenType::Let, "foobar"),))
-                ],
-            }
-        );
-
-        lexer = Lexer::new("let x 5; let 10; let 83838383;");
-        parser = Parser::new(lexer);
-        let _program = parser.parse_program();
-        check_parser_errors(&parser);
         assert_eq!(parser.errors.len(), 3);
+        assert!(program.is_some());
     }
 
     #[test]
     fn return_statements() {
+        let test_cases = [(
+            "return 5; return 10; return add(10);",
+            vec![
+                Statement::Return(ReturnStatement::new(Expression::IntegerLiteral(
+                    IntegerLiteral::new(5),
+                ))),
+                Statement::Return(ReturnStatement::new(Expression::IntegerLiteral(
+                    IntegerLiteral::new(10),
+                ))),
+                Statement::Return(ReturnStatement::new(Expression::CallExpression(
+                    CallExpression::new(
+                        Expression::Identifier(Identifier::new(TokenType::Ident, "add")),
+                        vec![Expression::IntegerLiteral(IntegerLiteral::new(10))],
+                    ),
+                ))),
+            ],
+        )];
+
+        for test in test_cases.iter() {
+            let lexer = Lexer::new(test.0);
+            let mut parser = Parser::new(lexer);
+            let program = parser.parse_program();
+            check_parser_errors(&parser);
+            assert_eq!(parser.errors.len(), 0);
+            assert!(program.is_some());
+            assert_eq!(program.unwrap().statements, test.1);
+        }
+
         let lexer = Lexer::new("return 5; return 10; return add(10);");
         let mut parser = Parser::new(lexer);
         let program = parser.parse_program();
@@ -322,20 +361,23 @@ mod tests {
                     )),
                 ))],
             ),
-            // TODO: Add this test when we add function call parser
+            (
-            // (
+                "!isGreaterThanZero( 2);",
-            //     "!isGreaterThanZero( 2);",
+                vec![Statement::ExpressionStatement(ExpressionStatement::new(
-            //     vec![Statement::ExpressionStatement(ExpressionStatement {
+                    Token::new(TokenType::Bang),
-            //         token: Token::new(TokenType::Bang),
+                    Expression::PrefixExpression(PrefixExpression::new(
-            //         expression: Expression::PrefixExpression(PrefixExpression::new(
+                        TokenType::Bang,
-            //             Token::new(TokenType::Bang),
+                        "!",
-            //             "!",
+                        Expression::CallExpression(CallExpression::new(
-            //             Expression::Identifier(Identifier::new(
+                            Expression::Identifier(Identifier::new(
-            //                 TokenType::Function,
+                                TokenType::Ident,
-            //                 "",
+                                "isGreaterThanZero",
-            //             )),
+                            )),
-            //         )),
+                            vec![Expression::IntegerLiteral(IntegerLiteral::new(2))],
-            //     })
+                        )),
+                    )),
+                ))],
+            ),
         ];
 
         for test in prefix_tests.iter() {
@@ -526,30 +568,24 @@ mod tests {
         let test_cases = [
             (
                 "true;",
-                Program {
+                vec![Statement::ExpressionStatement(ExpressionStatement::new(
-                    statements: vec![Statement::ExpressionStatement(ExpressionStatement::new(
+                    Token::new(TokenType::True),
-                        Token::new(TokenType::True),
+                    Expression::BooleanExpression(BooleanExpression::new(TokenType::True)),
-                        Expression::BooleanExpression(BooleanExpression::new(TokenType::True)),
+                ))],
-                    ))],
-                },
             ),
             (
                 "false;",
-                Program {
+                vec![Statement::ExpressionStatement(ExpressionStatement::new(
-                    statements: vec![Statement::ExpressionStatement(ExpressionStatement::new(
+                    Token::new(TokenType::False),
-                        Token::new(TokenType::False),
+                    Expression::BooleanExpression(BooleanExpression::new(TokenType::False)),
-                        Expression::BooleanExpression(BooleanExpression::new(TokenType::False)),
+                ))],
-                    ))],
-                },
             ),
             (
                 "let foobar = true;",
-                Program {
+                vec![Statement::Let(LetStatement::with_value(
-                    statements: vec![Statement::Let(LetStatement::with_value(
+                    Identifier::new(TokenType::Let, "foobar"),
-                        Identifier::new(TokenType::Let, "foobar"),
+                    Expression::BooleanExpression(BooleanExpression::new(TokenType::True)),
-                        None, // TODO: fix this when we complete parsing of let statements
+                ))],
-                    ))],
-                },
             ),
         ];
 
@@ -560,33 +596,31 @@ mod tests {
             check_parser_errors(&parser);
             assert_eq!(parser.errors.len(), 0);
             assert!(program.is_some());
-            assert_eq!(program.unwrap(), test.1);
+            assert_eq!(program.unwrap().statements, test.1);
         }
     }
     #[test]
     fn if_expression() {
         let test_cases = [(
             "if (x > y) { x };",
-            Program {
+            vec![Statement::ExpressionStatement(ExpressionStatement::new(
-                statements: vec![Statement::ExpressionStatement(ExpressionStatement::new(
+                Token::new(TokenType::If),
-                    Token::new(TokenType::If),
+                Expression::IfExpression(IfExpression::new(
-                    Expression::IfExpression(IfExpression::new(
+                    Expression::InfixExpression(InfixExpression::new(
-                        Expression::InfixExpression(InfixExpression::new(
+                        TokenType::GreaterThan,
-                            TokenType::GreaterThan,
+                        Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
-                            Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
+                        ">",
-                            ">",
+                        Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
-                            Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
-                        )),
-                        BlockStatement::new(vec![Statement::ExpressionStatement(
-                            ExpressionStatement::new(
-                                Token::with_value(TokenType::Ident, "x"),
-                                Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
-                            ),
-                        )]),
-                        None,
                     )),
-                ))],
+                    BlockStatement::new(vec![Statement::ExpressionStatement(
-            },
+                        ExpressionStatement::new(
+                            Token::with_value(TokenType::Ident, "x"),
+                            Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
+                        ),
+                    )]),
+                    None,
+                )),
+            ))],
         )];
 
         for test in test_cases.iter() {
@@ -596,38 +630,36 @@ mod tests {
             check_parser_errors(&parser);
             assert_eq!(parser.errors.len(), 0);
             assert!(program.is_some());
-            assert_eq!(program.unwrap(), test.1);
+            assert_eq!(program.unwrap().statements, test.1);
         }
     }
     #[test]
     fn if_else_expression() {
         let test_cases = [(
             "if (x > y) { x } else { y };",
-            Program {
+            vec![Statement::ExpressionStatement(ExpressionStatement::new(
-                statements: vec![Statement::ExpressionStatement(ExpressionStatement::new(
+                Token::new(TokenType::If),
-                    Token::new(TokenType::If),
+                Expression::IfExpression(IfExpression::new(
-                    Expression::IfExpression(IfExpression::new(
+                    Expression::InfixExpression(InfixExpression::new(
-                        Expression::InfixExpression(InfixExpression::new(
+                        TokenType::GreaterThan,
-                            TokenType::GreaterThan,
+                        Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
-                            Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
+                        ">",
-                            ">",
+                        Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
-                            Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
-                        )),
-                        BlockStatement::new(vec![Statement::ExpressionStatement(
-                            ExpressionStatement::new(
-                                Token::with_value(TokenType::Ident, "x"),
-                                Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
-                            ),
-                        )]),
-                        Some(BlockStatement::new(vec![Statement::ExpressionStatement(
-                            ExpressionStatement::new(
-                                Token::with_value(TokenType::Ident, "y"),
-                                Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
-                            ),
-                        )])),
                     )),
-                ))],
+                    BlockStatement::new(vec![Statement::ExpressionStatement(
-            },
+                        ExpressionStatement::new(
+                            Token::with_value(TokenType::Ident, "x"),
+                            Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
+                        ),
+                    )]),
+                    Some(BlockStatement::new(vec![Statement::ExpressionStatement(
+                        ExpressionStatement::new(
+                            Token::with_value(TokenType::Ident, "y"),
+                            Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
+                        ),
+                    )])),
+                )),
+            ))],
         )];
 
         for test in test_cases.iter() {
@@ -637,13 +669,12 @@ mod tests {
             check_parser_errors(&parser);
             assert_eq!(parser.errors.len(), 0);
             assert!(program.is_some());
-            assert_eq!(program.unwrap(), test.1);
+            assert_eq!(program.unwrap().statements, test.1);
         }
     }
 
     #[test]
     fn function_literal_parsing() {
-        // TODO: Figure out why are there inconsistencies in BlockStatements Token
         let test_cases = [
             (
                 "fn(a,b) {x + y;}",
@@ -767,7 +798,6 @@ mod tests {
     }
     #[test]
     fn call_expression_parsing() {
-        // TODO: Figure out why are there inconsistencies in BlockStatements Token
         let test_cases = [
             (
                 "add(1, 2 * 3, 4 + 5);",
@@ -930,7 +960,6 @@ mod tests {
     }
     #[test]
     fn call_expression_parsing_string() {
-        // TODO: Figure out why are there inconsistencies in BlockStatements Token
         let test_cases = [
             ("add(1, 2 * 3, 4 + 5);", "add(1, (2 * 3), (4 + 5))"),
             (