Implement parser using Pratt parsing
This commit is contained in:
cdo committed
1 parent
aad169c24f
commit
a9f737a1ad
3 files changed
+499
No files matched your search
@@ -3,3 +3,4 @@ pub mod expression;
|
||||
pub mod binding;
|
||||
|
||||
pub mod lex;
|
||||
pub mod parse;
|
||||
+487
@@ -0,0 +1,487 @@
|
||||
use crate::token::Token;
|
||||
use crate::expression::Expression;
|
||||
use crate::binding::{Binding, Power};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
Io(std::io::Error),
|
||||
InvalidSyntax,
|
||||
}
|
||||
|
||||
pub fn parse<I>(input: I) -> Result<Expression, Error>
|
||||
where I: Iterator<Item = Result<Token, std::io::Error>>
|
||||
{
|
||||
expr(&mut input.peekable(), Binding::left(Power::Minimum))
|
||||
}
|
||||
|
||||
fn expr<I>(
|
||||
input: &mut std::iter::Peekable<I>,
|
||||
limit: Binding) -> Result<Expression, Error>
|
||||
where I: Iterator<Item = Result<Token, std::io::Error>>
|
||||
{
|
||||
let t1 = input.next()
|
||||
.ok_or(Error::InvalidSyntax)?
|
||||
.map_err(|err| Error::Io(err))?;
|
||||
let mut left = nud(input, t1)?;
|
||||
|
||||
loop {
|
||||
match input.peek() {
|
||||
Some(Ok(op)) => {
|
||||
if Binding::of_operator(op) > limit {
|
||||
let op_copy = op.clone();
|
||||
left = led(input, left, op_copy)?;
|
||||
} else {
|
||||
break Ok(left)
|
||||
}
|
||||
},
|
||||
Some(Err(_)) => {
|
||||
let err = input.next().unwrap().unwrap_err();
|
||||
break Err(Error::Io(err));
|
||||
},
|
||||
None => break Ok(left),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn nud<I>(
|
||||
input: &mut std::iter::Peekable<I>,
|
||||
token: Token) -> Result<Expression, Error>
|
||||
where I: Iterator<Item = Result<Token, std::io::Error>>
|
||||
{
|
||||
match token {
|
||||
Token::Symbol(c) => Ok(Expression::Symbol(c)),
|
||||
Token::Integer(x) => Ok(Expression::Integer(x as i32)),
|
||||
Token::Plus => expr(input, Binding::right(Power::Multiplication)),
|
||||
Token::Minus => {
|
||||
let operand =
|
||||
expr(input, Binding::right(Power::Multiplication))?;
|
||||
Ok(match operand {
|
||||
Expression::Integer(x) => Expression::Integer(-x),
|
||||
_ => Expression::product(Expression::Integer(-1), operand),
|
||||
})
|
||||
}
|
||||
Token::OpenParen => {
|
||||
let operand = expr(input, Binding::left(Power::Minimum))?;
|
||||
match input.next() {
|
||||
Some(Ok(Token::CloseParen)) => Ok(operand),
|
||||
_ => Err(Error::InvalidSyntax),
|
||||
}
|
||||
}
|
||||
_ => Err(Error::InvalidSyntax),
|
||||
}
|
||||
}
|
||||
|
||||
fn led<I>(
|
||||
input: &mut std::iter::Peekable<I>,
|
||||
left: Expression,
|
||||
op: Token) -> Result<Expression, Error>
|
||||
where I: Iterator<Item = Result<Token, std::io::Error>>
|
||||
{
|
||||
if op.starts_expression() {
|
||||
let right = expr(input, Binding::of_operator(&op))?;
|
||||
Ok(Expression::product(left, right))
|
||||
} else {
|
||||
input.next();
|
||||
let right = expr(input, Binding::of_operator(&op))?;
|
||||
Ok(match op {
|
||||
Token::Plus => Expression::sum(left, right),
|
||||
Token::Minus => Expression::difference(left, right),
|
||||
Token::Times => Expression::product(left, right),
|
||||
Token::Divide => Expression::quotient(left, right),
|
||||
Token::Caret => Expression::exponent(left, right),
|
||||
_ => unreachable!(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn symbol_token_parses_to_symbol() {
|
||||
let tokens = vec![ Token::Symbol('x') ];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(expr, Expression::Symbol('x'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integer_token_parses_to_integer() {
|
||||
let tokens = vec![ Token::Integer(42) ];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(expr, Expression::Integer(42));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_plus_y_parses_as_sum() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::Plus,
|
||||
Token::Symbol('y'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::sum(
|
||||
Expression::Symbol('x'), Expression::Symbol('y')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_times_y_parses_as_product() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::Times,
|
||||
Token::Symbol('y'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::product(
|
||||
Expression::Symbol('x'), Expression::Symbol('y')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_plus_y_times_z_parses_as_sum_of_product() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::Plus,
|
||||
Token::Symbol('y'),
|
||||
Token::Times,
|
||||
Token::Symbol('z'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::sum(
|
||||
Expression::Symbol('x'),
|
||||
Expression::product(
|
||||
Expression::Symbol('y'), Expression::Symbol('z'))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_times_y_plus_z_parses_as_sum_of_product() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::Times,
|
||||
Token::Symbol('y'),
|
||||
Token::Plus,
|
||||
Token::Symbol('z'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::sum(
|
||||
Expression::product(
|
||||
Expression::Symbol('x'), Expression::Symbol('y')),
|
||||
Expression::Symbol('z')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_caret_y_times_z_parses_as_product_of_exponent() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::Caret,
|
||||
Token::Symbol('y'),
|
||||
Token::Times,
|
||||
Token::Symbol('z'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::product(
|
||||
Expression::exponent(
|
||||
Expression::Symbol('x'), Expression::Symbol('y')),
|
||||
Expression::Symbol('z')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_minus_y_minus_z_parses_as_left_associative() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::Minus,
|
||||
Token::Symbol('y'),
|
||||
Token::Minus,
|
||||
Token::Symbol('z'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::difference(
|
||||
Expression::difference(
|
||||
Expression::Symbol('x'), Expression::Symbol('y')),
|
||||
Expression::Symbol('z')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_divide_y_divide_z_parses_as_left_associative() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::Divide,
|
||||
Token::Symbol('y'),
|
||||
Token::Divide,
|
||||
Token::Symbol('z'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::quotient(
|
||||
Expression::quotient(
|
||||
Expression::Symbol('x'), Expression::Symbol('y')),
|
||||
Expression::Symbol('z')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_caret_y_caret_z_parses_as_right_associative() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::Caret,
|
||||
Token::Symbol('y'),
|
||||
Token::Caret,
|
||||
Token::Symbol('z'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::exponent(
|
||||
Expression::Symbol('x'),
|
||||
Expression::exponent(
|
||||
Expression::Symbol('y'), Expression::Symbol('z')),));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minus_6_parses_as_integer() {
|
||||
let tokens = vec![ Token::Minus, Token::Integer(6) ];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(expr, Expression::Integer(-6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minus_x_parses_as_product() {
|
||||
let tokens = vec![ Token::Minus, Token::Symbol('x') ];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::product(
|
||||
Expression::Integer(-1), Expression::Symbol('x')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plus_x_parses_as_symbol() {
|
||||
let tokens = vec![ Token::Plus, Token::Symbol('x') ];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(expr, Expression::Symbol('x'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minus_x_times_y_parses_with_higher_precedence_for_unary_minus() {
|
||||
let tokens = vec![
|
||||
Token::Minus,
|
||||
Token::Symbol('x'),
|
||||
Token::Times,
|
||||
Token::Symbol('y'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::product(
|
||||
Expression::product(
|
||||
Expression::Integer(-1), Expression::Symbol('x')),
|
||||
Expression::Symbol('y')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minus_x_caret_y_parses_with_higher_precedence_for_caret() {
|
||||
let tokens = vec![
|
||||
Token::Minus,
|
||||
Token::Symbol('x'),
|
||||
Token::Caret,
|
||||
Token::Symbol('y'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::product(
|
||||
Expression::Integer(-1),
|
||||
Expression::exponent(
|
||||
Expression::Symbol('x'), Expression::Symbol('y'))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_times_oparen_y_plus_z_cparen_parses_as_product_of_sum() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::Times,
|
||||
Token::OpenParen,
|
||||
Token::Symbol('y'),
|
||||
Token::Plus,
|
||||
Token::Symbol('z'),
|
||||
Token::CloseParen,
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::product(
|
||||
Expression::Symbol('x'),
|
||||
Expression::sum(
|
||||
Expression::Symbol('y'), Expression::Symbol('z'))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_y_parses_as_product() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::Symbol('y'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::product(
|
||||
Expression::Symbol('x'), Expression::Symbol('y')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_2_x_parses_as_product() {
|
||||
let tokens = vec![
|
||||
Token::Integer(2),
|
||||
Token::Symbol('x'),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::product(
|
||||
Expression::Integer(2), Expression::Symbol('x')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_2_parses_as_product() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::Integer(2),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::product(
|
||||
Expression::Symbol('x'), Expression::Integer(2)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complex_expression_1() {
|
||||
let tokens = vec![
|
||||
Token::Integer(2),
|
||||
Token::Symbol('x'),
|
||||
Token::Caret,
|
||||
Token::Integer(3),
|
||||
Token::Minus,
|
||||
Token::Symbol('y'),
|
||||
Token::Divide,
|
||||
Token::Symbol('z'),
|
||||
Token::Caret,
|
||||
Token::Integer(2),
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::difference(
|
||||
Expression::product(
|
||||
Expression::Integer(2),
|
||||
Expression::exponent(
|
||||
Expression::Symbol('x'),
|
||||
Expression::Integer(3))),
|
||||
Expression::quotient(
|
||||
Expression::Symbol('y'),
|
||||
Expression::exponent(
|
||||
Expression::Symbol('z'),
|
||||
Expression::Integer(2)))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complex_expression_2() {
|
||||
let tokens = vec![
|
||||
Token::Symbol('x'),
|
||||
Token::OpenParen,
|
||||
Token::Symbol('x'),
|
||||
Token::Plus,
|
||||
Token::Symbol('y'),
|
||||
Token::Minus,
|
||||
Token::Integer(1),
|
||||
Token::CloseParen,
|
||||
Token::Plus,
|
||||
Token::Symbol('y'),
|
||||
Token::Caret,
|
||||
Token::OpenParen,
|
||||
Token::Integer(3),
|
||||
Token::Minus,
|
||||
Token::Symbol('x'),
|
||||
Token::CloseParen,
|
||||
];
|
||||
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
|
||||
assert_eq!(
|
||||
expr,
|
||||
Expression::sum(
|
||||
Expression::product(
|
||||
Expression::Symbol('x'),
|
||||
Expression::difference(
|
||||
Expression::sum(
|
||||
Expression::Symbol('x'),
|
||||
Expression::Symbol('y')),
|
||||
Expression::Integer(1))),
|
||||
Expression::exponent(
|
||||
Expression::Symbol('y'),
|
||||
Expression::difference(
|
||||
Expression::Integer(3),
|
||||
Expression::Symbol('x')))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unclosed_paren_is_invalid_syntax() {
|
||||
let tokens = vec![
|
||||
Token::OpenParen,
|
||||
Token::Symbol('x'),
|
||||
Token::Plus,
|
||||
Token::Symbol('y'),
|
||||
];
|
||||
let result = parse(tokens.into_iter().map(|t| Ok(t)));
|
||||
std::assert_matches!(result, Err(Error::InvalidSyntax));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn close_paren_x_is_invalid_syntax() {
|
||||
let tokens = vec![
|
||||
Token::CloseParen,
|
||||
Token::Symbol('x'),
|
||||
];
|
||||
let result = parse(tokens.into_iter().map(|t| Ok(t)));
|
||||
std::assert_matches!(result, Err(Error::InvalidSyntax));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn times_x_is_invalid_syntax() {
|
||||
let tokens = vec![
|
||||
Token::Times,
|
||||
Token::Symbol('x'),
|
||||
];
|
||||
let result = parse(tokens.into_iter().map(|t| Ok(t)));
|
||||
std::assert_matches!(result, Err(Error::InvalidSyntax));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn divide_x_is_invalid_syntax() {
|
||||
let tokens = vec![
|
||||
Token::Divide,
|
||||
Token::Symbol('x'),
|
||||
];
|
||||
let result = parse(tokens.into_iter().map(|t| Ok(t)));
|
||||
std::assert_matches!(result, Err(Error::InvalidSyntax));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caret_x_is_invalid_syntax() {
|
||||
let tokens = vec![
|
||||
Token::Caret,
|
||||
Token::Symbol('x'),
|
||||
];
|
||||
let result = parse(tokens.into_iter().map(|t| Ok(t)));
|
||||
std::assert_matches!(result, Err(Error::InvalidSyntax));
|
||||
}
|
||||
}
|
||||
@@ -10,3 +10,14 @@ pub enum Token {
|
||||
Divide,
|
||||
Caret,
|
||||
}
|
||||
|
||||
impl Token {
|
||||
pub fn starts_expression(&self) -> bool {
|
||||
match self {
|
||||
Token::OpenParen => true,
|
||||
Token::Symbol(_) => true,
|
||||
Token::Integer(_) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user