ishan/monkey
1
0
Fork 0
Code Releases Activity

Completed Let/Return Statement parser

Browse Source 
1. Completed Expression parsing in Let/Return Statement Parsers
2. Removed most of the TODOs
3. Added more tests
This commit is contained in:
Ishan Jain 2020-01-20 17:38:58 +05:30
parent a1c316a43e
commit 0833ba3448
2 changed files with 155 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
src/parser/ast/mod.rs
View File

@ -1,4 +1,3 @@
// TODO: Maybe implement String method to pretty print all AST nodes
use {
crate::{
lexer::{Token, TokenType},
@ -71,26 +70,26 @@ impl LetStatement {
pub fn new(name: Identifier) -> Self {
Self { name, value: None }
}
pub fn with_value(name: Identifier, value: Option<Expression>) -> Self {
Self { name, value }
pub fn with_value(name: Identifier, value: Expression) -> Self {
Self {
name,
value: Some(value),
}
}
// TODO: Implement code to parse let statement
pub fn parse(parser: &mut Parser) -> Option<Self> {
let mut stmt = LetStatement {
name: Identifier::new(TokenType::Let, "placeholder_value"),
value: None,
};
let ident = parser.expect_peek(TokenType::Ident)?;
stmt.name.value = ident.literal?;
parser.expect_peek(TokenType::Assign)?;
let ntoken = parser.lexer.next()?;
let expr = Expression::parse(parser, ntoken, ExpressionPriority::Lowest);
// TODO: Right now, We are just skipping over all the expressions
// That'll come later
// Also, Right now, It hangs forever in case there is no semicolon at the end
while parser.lexer.next() != Some(Token::new(TokenType::Semicolon)) {}
parser.expect_peek(TokenType::Semicolon);
Some(stmt)
Some(Self {
name: Identifier::new(TokenType::Let, &ident.literal?),
value: expr,
})
}
const fn token_literal() -> &'static str {
@ -117,12 +116,17 @@ pub struct ReturnStatement {
}
impl ReturnStatement {
pub fn new(expr: Expression) -> Self {
ReturnStatement {
return_value: Some(expr),
}
}
fn parse(parser: &mut Parser) -> Option<Self> {
let stmt = ReturnStatement {
return_value: Some(Expression::None),
};
while parser.lexer.next() != Some(Token::new(TokenType::Semicolon)) {}
return Some(stmt);
let token = parser.lexer.next()?;
let expr = Expression::parse(parser, token, ExpressionPriority::Lowest);
parser.expect_peek(TokenType::Semicolon);
return Some(ReturnStatement { return_value: expr });
}
const fn token_literal() -> &'static str {
@ -193,8 +197,6 @@ pub enum Expression {
IfExpression(IfExpression),
FunctionExpression(FunctionLiteral),
CallExpression(CallExpression),
// TODO: Temporary placeholder value. Should be removed once this section is done
None,
}
impl Expression {
@ -249,7 +251,6 @@ impl Display for Expression {
Expression::IfExpression(v) => v.to_string(),
Expression::FunctionExpression(v) => v.to_string(),
Expression::CallExpression(v) => v.to_string(),
Expression::None => "None".into(),
};
f.write_str(&value)

233
src/parser/mod.rs
View File

@ -15,7 +15,6 @@ type PrefixParseFn = fn(&mut Parser, token: Token) -> Option<Expression>;
type InfixParseFn = fn(&mut Parser, Token, Expression) -> Option<Expression>;
//TODO: Add Parser tracing so we can easily figure out the call stack hierarchy
lazy_static! {
static ref PRECEDENCE_MAP: HashMap<TokenType, ExpressionPriority> = {
let mut m = HashMap::new();
@ -186,34 +185,74 @@ mod tests {
#[test]
fn let_statements() {
let mut lexer = Lexer::new("let x =5;let y=10; let foobar=538383;");
let test_cases = [(
"let x =5;let y=10; let foobar=538383;",
vec![
Statement::Let(LetStatement::with_value(
Identifier::new(TokenType::Let, "x"),
Expression::IntegerLiteral(IntegerLiteral::new(5)),
)),
Statement::Let(LetStatement::with_value(
Identifier::new(TokenType::Let, "y"),
Expression::IntegerLiteral(IntegerLiteral::new(10)),
)),
Statement::Let(LetStatement::with_value(
Identifier::new(TokenType::Let, "foobar"),
Expression::IntegerLiteral(IntegerLiteral::new(538383)),
)),
],
)];
for test in test_cases.iter() {
let lexer = Lexer::new(test.0);
let mut parser = Parser::new(lexer);
let program = parser.parse_program();
check_parser_errors(&parser);
assert_eq!(parser.errors.len(), 0);
assert!(program.is_some());
assert_eq!(program.unwrap().statements, test.1);
}
let fail_case = "let x 5; let 10; let 83838383;";
let lexer = Lexer::new(fail_case);
let mut parser = Parser::new(lexer);
let program = parser.parse_program();
check_parser_errors(&parser);
assert_eq!(parser.errors.len(), 0);
assert!(program.is_some());
let program = program.unwrap();
assert_eq!(program.statements.len(), 3);
assert_eq!(
program,
Program {
statements: vec![
Statement::Let(LetStatement::new(Identifier::new(TokenType::Let, "x"),)),
Statement::Let(LetStatement::new(Identifier::new(TokenType::Let, "y"),)),
Statement::Let(LetStatement::new(Identifier::new(TokenType::Let, "foobar"),))
],
}
);
lexer = Lexer::new("let x 5; let 10; let 83838383;");
parser = Parser::new(lexer);
let _program = parser.parse_program();
check_parser_errors(&parser);
assert_eq!(parser.errors.len(), 3);
assert!(program.is_some());
}
#[test]
fn return_statements() {
let test_cases = [(
"return 5; return 10; return add(10);",
vec![
Statement::Return(ReturnStatement::new(Expression::IntegerLiteral(
IntegerLiteral::new(5),
))),
Statement::Return(ReturnStatement::new(Expression::IntegerLiteral(
IntegerLiteral::new(10),
))),
Statement::Return(ReturnStatement::new(Expression::CallExpression(
CallExpression::new(
Expression::Identifier(Identifier::new(TokenType::Ident, "add")),
vec![Expression::IntegerLiteral(IntegerLiteral::new(10))],
),
))),
],
)];
for test in test_cases.iter() {
let lexer = Lexer::new(test.0);
let mut parser = Parser::new(lexer);
let program = parser.parse_program();
check_parser_errors(&parser);
assert_eq!(parser.errors.len(), 0);
assert!(program.is_some());
assert_eq!(program.unwrap().statements, test.1);
}
let lexer = Lexer::new("return 5; return 10; return add(10);");
let mut parser = Parser::new(lexer);
let program = parser.parse_program();
@ -322,20 +361,23 @@ mod tests {
)),
))],
),
// TODO: Add this test when we add function call parser
// (
// "!isGreaterThanZero( 2);",
// vec![Statement::ExpressionStatement(ExpressionStatement {
// token: Token::new(TokenType::Bang),
// expression: Expression::PrefixExpression(PrefixExpression::new(
// Token::new(TokenType::Bang),
// "!",
// Expression::Identifier(Identifier::new(
// TokenType::Function,
// "",
// )),
// )),
// })
(
"!isGreaterThanZero( 2);",
vec![Statement::ExpressionStatement(ExpressionStatement::new(
Token::new(TokenType::Bang),
Expression::PrefixExpression(PrefixExpression::new(
TokenType::Bang,
"!",
Expression::CallExpression(CallExpression::new(
Expression::Identifier(Identifier::new(
TokenType::Ident,
"isGreaterThanZero",
)),
vec![Expression::IntegerLiteral(IntegerLiteral::new(2))],
)),
)),
))],
),
];
for test in prefix_tests.iter() {
@ -526,30 +568,24 @@ mod tests {
let test_cases = [
(
"true;",
Program {
statements: vec![Statement::ExpressionStatement(ExpressionStatement::new(
Token::new(TokenType::True),
Expression::BooleanExpression(BooleanExpression::new(TokenType::True)),
))],
},
vec![Statement::ExpressionStatement(ExpressionStatement::new(
Token::new(TokenType::True),
Expression::BooleanExpression(BooleanExpression::new(TokenType::True)),
))],
),
(
"false;",
Program {
statements: vec![Statement::ExpressionStatement(ExpressionStatement::new(
Token::new(TokenType::False),
Expression::BooleanExpression(BooleanExpression::new(TokenType::False)),
))],
},
vec![Statement::ExpressionStatement(ExpressionStatement::new(
Token::new(TokenType::False),
Expression::BooleanExpression(BooleanExpression::new(TokenType::False)),
))],
),
(
"let foobar = true;",
Program {
statements: vec![Statement::Let(LetStatement::with_value(
Identifier::new(TokenType::Let, "foobar"),
None, // TODO: fix this when we complete parsing of let statements
))],
},
vec![Statement::Let(LetStatement::with_value(
Identifier::new(TokenType::Let, "foobar"),
Expression::BooleanExpression(BooleanExpression::new(TokenType::True)),
))],
),
];
@ -560,33 +596,31 @@ mod tests {
check_parser_errors(&parser);
assert_eq!(parser.errors.len(), 0);
assert!(program.is_some());
assert_eq!(program.unwrap(), test.1);
assert_eq!(program.unwrap().statements, test.1);
}
}
#[test]
fn if_expression() {
let test_cases = [(
"if (x > y) { x };",
Program {
statements: vec![Statement::ExpressionStatement(ExpressionStatement::new(
Token::new(TokenType::If),
Expression::IfExpression(IfExpression::new(
Expression::InfixExpression(InfixExpression::new(
TokenType::GreaterThan,
Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
">",
Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
)),
BlockStatement::new(vec![Statement::ExpressionStatement(
ExpressionStatement::new(
Token::with_value(TokenType::Ident, "x"),
Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
),
)]),
None,
vec![Statement::ExpressionStatement(ExpressionStatement::new(
Token::new(TokenType::If),
Expression::IfExpression(IfExpression::new(
Expression::InfixExpression(InfixExpression::new(
TokenType::GreaterThan,
Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
">",
Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
)),
))],
},
BlockStatement::new(vec![Statement::ExpressionStatement(
ExpressionStatement::new(
Token::with_value(TokenType::Ident, "x"),
Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
),
)]),
None,
)),
))],
)];
for test in test_cases.iter() {
@ -596,38 +630,36 @@ mod tests {
check_parser_errors(&parser);
assert_eq!(parser.errors.len(), 0);
assert!(program.is_some());
assert_eq!(program.unwrap(), test.1);
assert_eq!(program.unwrap().statements, test.1);
}
}
#[test]
fn if_else_expression() {
let test_cases = [(
"if (x > y) { x } else { y };",
Program {
statements: vec![Statement::ExpressionStatement(ExpressionStatement::new(
Token::new(TokenType::If),
Expression::IfExpression(IfExpression::new(
Expression::InfixExpression(InfixExpression::new(
TokenType::GreaterThan,
Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
">",
Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
)),
BlockStatement::new(vec![Statement::ExpressionStatement(
ExpressionStatement::new(
Token::with_value(TokenType::Ident, "x"),
Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
),
)]),
Some(BlockStatement::new(vec![Statement::ExpressionStatement(
ExpressionStatement::new(
Token::with_value(TokenType::Ident, "y"),
Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
),
)])),
vec![Statement::ExpressionStatement(ExpressionStatement::new(
Token::new(TokenType::If),
Expression::IfExpression(IfExpression::new(
Expression::InfixExpression(InfixExpression::new(
TokenType::GreaterThan,
Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
">",
Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
)),
))],
},
BlockStatement::new(vec![Statement::ExpressionStatement(
ExpressionStatement::new(
Token::with_value(TokenType::Ident, "x"),
Expression::Identifier(Identifier::new(TokenType::Ident, "x")),
),
)]),
Some(BlockStatement::new(vec![Statement::ExpressionStatement(
ExpressionStatement::new(
Token::with_value(TokenType::Ident, "y"),
Expression::Identifier(Identifier::new(TokenType::Ident, "y")),
),
)])),
)),
))],
)];
for test in test_cases.iter() {
@ -637,13 +669,12 @@ mod tests {
check_parser_errors(&parser);
assert_eq!(parser.errors.len(), 0);
assert!(program.is_some());
assert_eq!(program.unwrap(), test.1);
assert_eq!(program.unwrap().statements, test.1);
}
}
#[test]
fn function_literal_parsing() {
// TODO: Figure out why are there inconsistencies in BlockStatements Token
let test_cases = [
(
"fn(a,b) {x + y;}",
@ -767,7 +798,6 @@ mod tests {
}
#[test]
fn call_expression_parsing() {
// TODO: Figure out why are there inconsistencies in BlockStatements Token
let test_cases = [
(
"add(1, 2 * 3, 4 + 5);",
@ -930,7 +960,6 @@ mod tests {
}
#[test]
fn call_expression_parsing_string() {
// TODO: Figure out why are there inconsistencies in BlockStatements Token
let test_cases = [
("add(1, 2 * 3, 4 + 5);", "add(1, (2 * 3), (4 + 5))"),
(