From 0833ba344856e971d2ce78f2c09f966c61708639 Mon Sep 17 00:00:00 2001 From: ishanjain28 Date: Mon, 20 Jan 2020 17:38:58 +0530 Subject: [PATCH] Completed Let/Return Statement parser 1. Completed Expression parsing in Let/Return Statement Parsers 2. Removed most of the TODOs 3. Added more tests --- src/parser/ast/mod.rs | 47 ++++----- src/parser/mod.rs | 233 ++++++++++++++++++++++++------------------ 2 files changed, 155 insertions(+), 125 deletions(-) diff --git a/src/parser/ast/mod.rs b/src/parser/ast/mod.rs index a68a417..fd878e3 100644 --- a/src/parser/ast/mod.rs +++ b/src/parser/ast/mod.rs @@ -1,4 +1,3 @@ -// TODO: Maybe implement String method to pretty print all AST nodes use { crate::{ lexer::{Token, TokenType}, @@ -71,26 +70,26 @@ impl LetStatement { pub fn new(name: Identifier) -> Self { Self { name, value: None } } - pub fn with_value(name: Identifier, value: Option) -> Self { - Self { name, value } + pub fn with_value(name: Identifier, value: Expression) -> Self { + Self { + name, + value: Some(value), + } } - // TODO: Implement code to parse let statement + pub fn parse(parser: &mut Parser) -> Option { - let mut stmt = LetStatement { - name: Identifier::new(TokenType::Let, "placeholder_value"), - value: None, - }; - let ident = parser.expect_peek(TokenType::Ident)?; - stmt.name.value = ident.literal?; + parser.expect_peek(TokenType::Assign)?; + let ntoken = parser.lexer.next()?; + let expr = Expression::parse(parser, ntoken, ExpressionPriority::Lowest); - // TODO: Right now, We are just skipping over all the expressions - // That'll come later - // Also, Right now, It hangs forever in case there is no semicolon at the end - while parser.lexer.next() != Some(Token::new(TokenType::Semicolon)) {} + parser.expect_peek(TokenType::Semicolon); - Some(stmt) + Some(Self { + name: Identifier::new(TokenType::Let, &ident.literal?), + value: expr, + }) } const fn token_literal() -> &'static str { @@ -117,12 +116,17 @@ pub struct ReturnStatement { } impl ReturnStatement { + pub fn new(expr: Expression) -> Self { + ReturnStatement { + return_value: Some(expr), + } + } + fn parse(parser: &mut Parser) -> Option { - let stmt = ReturnStatement { - return_value: Some(Expression::None), - }; - while parser.lexer.next() != Some(Token::new(TokenType::Semicolon)) {} - return Some(stmt); + let token = parser.lexer.next()?; + let expr = Expression::parse(parser, token, ExpressionPriority::Lowest); + parser.expect_peek(TokenType::Semicolon); + return Some(ReturnStatement { return_value: expr }); } const fn token_literal() -> &'static str { @@ -193,8 +197,6 @@ pub enum Expression { IfExpression(IfExpression), FunctionExpression(FunctionLiteral), CallExpression(CallExpression), - // TODO: Temporary placeholder value. Should be removed once this section is done - None, } impl Expression { @@ -249,7 +251,6 @@ impl Display for Expression { Expression::IfExpression(v) => v.to_string(), Expression::FunctionExpression(v) => v.to_string(), Expression::CallExpression(v) => v.to_string(), - Expression::None => "None".into(), }; f.write_str(&value) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index cb24111..c702c9a 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -15,7 +15,6 @@ type PrefixParseFn = fn(&mut Parser, token: Token) -> Option; type InfixParseFn = fn(&mut Parser, Token, Expression) -> Option; //TODO: Add Parser tracing so we can easily figure out the call stack hierarchy - lazy_static! { static ref PRECEDENCE_MAP: HashMap = { let mut m = HashMap::new(); @@ -186,34 +185,74 @@ mod tests { #[test] fn let_statements() { - let mut lexer = Lexer::new("let x =5;let y=10; let foobar=538383;"); + let test_cases = [( + "let x =5;let y=10; let foobar=538383;", + vec![ + Statement::Let(LetStatement::with_value( + Identifier::new(TokenType::Let, "x"), + Expression::IntegerLiteral(IntegerLiteral::new(5)), + )), + Statement::Let(LetStatement::with_value( + Identifier::new(TokenType::Let, "y"), + Expression::IntegerLiteral(IntegerLiteral::new(10)), + )), + Statement::Let(LetStatement::with_value( + Identifier::new(TokenType::Let, "foobar"), + Expression::IntegerLiteral(IntegerLiteral::new(538383)), + )), + ], + )]; + + for test in test_cases.iter() { + let lexer = Lexer::new(test.0); + let mut parser = Parser::new(lexer); + let program = parser.parse_program(); + check_parser_errors(&parser); + assert_eq!(parser.errors.len(), 0); + assert!(program.is_some()); + assert_eq!(program.unwrap().statements, test.1); + } + + let fail_case = "let x 5; let 10; let 83838383;"; + + let lexer = Lexer::new(fail_case); let mut parser = Parser::new(lexer); let program = parser.parse_program(); check_parser_errors(&parser); - assert_eq!(parser.errors.len(), 0); - assert!(program.is_some()); - let program = program.unwrap(); - assert_eq!(program.statements.len(), 3); - assert_eq!( - program, - Program { - statements: vec![ - Statement::Let(LetStatement::new(Identifier::new(TokenType::Let, "x"),)), - Statement::Let(LetStatement::new(Identifier::new(TokenType::Let, "y"),)), - Statement::Let(LetStatement::new(Identifier::new(TokenType::Let, "foobar"),)) - ], - } - ); - - lexer = Lexer::new("let x 5; let 10; let 83838383;"); - parser = Parser::new(lexer); - let _program = parser.parse_program(); - check_parser_errors(&parser); assert_eq!(parser.errors.len(), 3); + assert!(program.is_some()); } #[test] fn return_statements() { + let test_cases = [( + "return 5; return 10; return add(10);", + vec![ + Statement::Return(ReturnStatement::new(Expression::IntegerLiteral( + IntegerLiteral::new(5), + ))), + Statement::Return(ReturnStatement::new(Expression::IntegerLiteral( + IntegerLiteral::new(10), + ))), + Statement::Return(ReturnStatement::new(Expression::CallExpression( + CallExpression::new( + Expression::Identifier(Identifier::new(TokenType::Ident, "add")), + vec![Expression::IntegerLiteral(IntegerLiteral::new(10))], + ), + ))), + ], + )]; + + for test in test_cases.iter() { + let lexer = Lexer::new(test.0); + let mut parser = Parser::new(lexer); + let program = parser.parse_program(); + check_parser_errors(&parser); + assert_eq!(parser.errors.len(), 0); + assert!(program.is_some()); + assert_eq!(program.unwrap().statements, test.1); + } + let lexer = Lexer::new("return 5; return 10; return add(10);"); let mut parser = Parser::new(lexer); let program = parser.parse_program(); @@ -322,20 +361,23 @@ mod tests { )), ))], ), - // TODO: Add this test when we add function call parser - // ( - // "!isGreaterThanZero( 2);", - // vec![Statement::ExpressionStatement(ExpressionStatement { - // token: Token::new(TokenType::Bang), - // expression: Expression::PrefixExpression(PrefixExpression::new( - // Token::new(TokenType::Bang), - // "!", - // Expression::Identifier(Identifier::new( - // TokenType::Function, - // "", - // )), - // )), - // }) + ( + "!isGreaterThanZero( 2);", + vec![Statement::ExpressionStatement(ExpressionStatement::new( + Token::new(TokenType::Bang), + Expression::PrefixExpression(PrefixExpression::new( + TokenType::Bang, + "!", + Expression::CallExpression(CallExpression::new( + Expression::Identifier(Identifier::new( + TokenType::Ident, + "isGreaterThanZero", + )), + vec![Expression::IntegerLiteral(IntegerLiteral::new(2))], + )), + )), + ))], + ), ]; for test in prefix_tests.iter() { @@ -526,30 +568,24 @@ mod tests { let test_cases = [ ( "true;", - Program { - statements: vec![Statement::ExpressionStatement(ExpressionStatement::new( - Token::new(TokenType::True), - Expression::BooleanExpression(BooleanExpression::new(TokenType::True)), - ))], - }, + vec![Statement::ExpressionStatement(ExpressionStatement::new( + Token::new(TokenType::True), + Expression::BooleanExpression(BooleanExpression::new(TokenType::True)), + ))], ), ( "false;", - Program { - statements: vec![Statement::ExpressionStatement(ExpressionStatement::new( - Token::new(TokenType::False), - Expression::BooleanExpression(BooleanExpression::new(TokenType::False)), - ))], - }, + vec![Statement::ExpressionStatement(ExpressionStatement::new( + Token::new(TokenType::False), + Expression::BooleanExpression(BooleanExpression::new(TokenType::False)), + ))], ), ( "let foobar = true;", - Program { - statements: vec![Statement::Let(LetStatement::with_value( - Identifier::new(TokenType::Let, "foobar"), - None, // TODO: fix this when we complete parsing of let statements - ))], - }, + vec![Statement::Let(LetStatement::with_value( + Identifier::new(TokenType::Let, "foobar"), + Expression::BooleanExpression(BooleanExpression::new(TokenType::True)), + ))], ), ]; @@ -560,33 +596,31 @@ mod tests { check_parser_errors(&parser); assert_eq!(parser.errors.len(), 0); assert!(program.is_some()); - assert_eq!(program.unwrap(), test.1); + assert_eq!(program.unwrap().statements, test.1); } } #[test] fn if_expression() { let test_cases = [( "if (x > y) { x };", - Program { - statements: vec![Statement::ExpressionStatement(ExpressionStatement::new( - Token::new(TokenType::If), - Expression::IfExpression(IfExpression::new( - Expression::InfixExpression(InfixExpression::new( - TokenType::GreaterThan, - Expression::Identifier(Identifier::new(TokenType::Ident, "x")), - ">", - Expression::Identifier(Identifier::new(TokenType::Ident, "y")), - )), - BlockStatement::new(vec![Statement::ExpressionStatement( - ExpressionStatement::new( - Token::with_value(TokenType::Ident, "x"), - Expression::Identifier(Identifier::new(TokenType::Ident, "x")), - ), - )]), - None, + vec![Statement::ExpressionStatement(ExpressionStatement::new( + Token::new(TokenType::If), + Expression::IfExpression(IfExpression::new( + Expression::InfixExpression(InfixExpression::new( + TokenType::GreaterThan, + Expression::Identifier(Identifier::new(TokenType::Ident, "x")), + ">", + Expression::Identifier(Identifier::new(TokenType::Ident, "y")), )), - ))], - }, + BlockStatement::new(vec![Statement::ExpressionStatement( + ExpressionStatement::new( + Token::with_value(TokenType::Ident, "x"), + Expression::Identifier(Identifier::new(TokenType::Ident, "x")), + ), + )]), + None, + )), + ))], )]; for test in test_cases.iter() { @@ -596,38 +630,36 @@ mod tests { check_parser_errors(&parser); assert_eq!(parser.errors.len(), 0); assert!(program.is_some()); - assert_eq!(program.unwrap(), test.1); + assert_eq!(program.unwrap().statements, test.1); } } #[test] fn if_else_expression() { let test_cases = [( "if (x > y) { x } else { y };", - Program { - statements: vec![Statement::ExpressionStatement(ExpressionStatement::new( - Token::new(TokenType::If), - Expression::IfExpression(IfExpression::new( - Expression::InfixExpression(InfixExpression::new( - TokenType::GreaterThan, - Expression::Identifier(Identifier::new(TokenType::Ident, "x")), - ">", - Expression::Identifier(Identifier::new(TokenType::Ident, "y")), - )), - BlockStatement::new(vec![Statement::ExpressionStatement( - ExpressionStatement::new( - Token::with_value(TokenType::Ident, "x"), - Expression::Identifier(Identifier::new(TokenType::Ident, "x")), - ), - )]), - Some(BlockStatement::new(vec![Statement::ExpressionStatement( - ExpressionStatement::new( - Token::with_value(TokenType::Ident, "y"), - Expression::Identifier(Identifier::new(TokenType::Ident, "y")), - ), - )])), + vec![Statement::ExpressionStatement(ExpressionStatement::new( + Token::new(TokenType::If), + Expression::IfExpression(IfExpression::new( + Expression::InfixExpression(InfixExpression::new( + TokenType::GreaterThan, + Expression::Identifier(Identifier::new(TokenType::Ident, "x")), + ">", + Expression::Identifier(Identifier::new(TokenType::Ident, "y")), )), - ))], - }, + BlockStatement::new(vec![Statement::ExpressionStatement( + ExpressionStatement::new( + Token::with_value(TokenType::Ident, "x"), + Expression::Identifier(Identifier::new(TokenType::Ident, "x")), + ), + )]), + Some(BlockStatement::new(vec![Statement::ExpressionStatement( + ExpressionStatement::new( + Token::with_value(TokenType::Ident, "y"), + Expression::Identifier(Identifier::new(TokenType::Ident, "y")), + ), + )])), + )), + ))], )]; for test in test_cases.iter() { @@ -637,13 +669,12 @@ mod tests { check_parser_errors(&parser); assert_eq!(parser.errors.len(), 0); assert!(program.is_some()); - assert_eq!(program.unwrap(), test.1); + assert_eq!(program.unwrap().statements, test.1); } } #[test] fn function_literal_parsing() { - // TODO: Figure out why are there inconsistencies in BlockStatements Token let test_cases = [ ( "fn(a,b) {x + y;}", @@ -767,7 +798,6 @@ mod tests { } #[test] fn call_expression_parsing() { - // TODO: Figure out why are there inconsistencies in BlockStatements Token let test_cases = [ ( "add(1, 2 * 3, 4 + 5);", @@ -930,7 +960,6 @@ mod tests { } #[test] fn call_expression_parsing_string() { - // TODO: Figure out why are there inconsistencies in BlockStatements Token let test_cases = [ ("add(1, 2 * 3, 4 + 5);", "add(1, (2 * 3), (4 + 5))"), (