Reorganise lex and parse under single read module

This commit is contained in:
cdo committed 2026-09-26 10:49:03 +01:00
1 parent df28276496
commit fb1a959bb7
4 files changed
+37 -28

No files matched your search

+182
View File
@@ -0,0 +1,182 @@
use crate::token::Token;
#[derive(Debug)]
enum State {
Init,
Integer(i32),
}
pub struct TokenSource<R> {
input: R,
state: State,
}
const RADIX: u32 = 10;
fn skip(c: char) -> bool {
return c.is_whitespace();
}
fn token_of(c: char) -> Token {
match c {
'(' => Token::OpenParen,
')' => Token::CloseParen,
'+' => Token::Plus,
'-' => Token::Minus,
'*' => Token::Times,
'/' => Token::Divide,
'^' => Token::Caret,
_ => Token::Symbol(c),
}
}
impl<R: std::io::BufRead> TokenSource<R> {
pub fn new(input: R) -> Self {
Self { input, state: State::Init }
}
fn handle_char_init(&mut self, c: char) -> Option<Token> {
self.input.consume(1);
if skip(c) {
None
} else if let Some(digit) = c.to_digit(RADIX) {
self.state = State::Integer(digit as i32);
None
} else {
Some(token_of(c))
}
}
fn handle_char_integer(&mut self, value: i32, c: char) -> Option<Token> {
if let Some(digit) = c.to_digit(RADIX) {
self.input.consume(1);
self.state = State::Integer(
(RADIX as i32) * value + digit as i32);
None
} else {
self.state = State::Init;
Some(Token::Integer(value))
}
}
fn dispatch_byte(&mut self, byte: u8) -> Option<Token> {
let c = byte as char;
match self.state {
State::Init => self.handle_char_init(c),
State::Integer(value) => self.handle_char_integer(value, c),
}
}
fn dispatch_eof(&mut self) -> Option<Token> {
match self.state {
State::Init => None,
State::Integer(value) => {
self.state = State::Init;
Some(Token::Integer(value))
}
}
}
}
impl<R: std::io::BufRead> Iterator for TokenSource<R> {
type Item = Result<Token, std::io::Error>;
fn next(&mut self) -> Option<Self::Item> {
loop {
match self.input.fill_buf() {
Ok(buf) => match buf.first() {
Some(&byte) => {
if let Some(token) = self.dispatch_byte(byte) {
break Some(Ok(token));
}
}
None => break self.dispatch_eof().map(|token| Ok(token))
}
Err(err) => break Some(Err(err))
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn empty_input_yields_no_tokens() {
let ts = TokenSource::new(std::io::Cursor::new(b""));
let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
assert!(tokens.is_empty());
}
#[test]
fn input_x_yields_one_symbol_token() {
let ts = TokenSource::new(std::io::Cursor::new(b"x"));
let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
assert_eq!(tokens, vec![Token::Symbol('x')]);
}
#[test]
fn input_xy_yeilds_two_symbol_tokens() {
let ts = TokenSource::new(std::io::Cursor::new(b"xy"));
let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
assert_eq!(tokens, vec![Token::Symbol('x'), Token::Symbol('y')]);
}
#[test]
fn input_12_yields_one_integer_token() {
let ts = TokenSource::new(std::io::Cursor::new(b"12"));
let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
assert_eq!(tokens, vec![Token::Integer(12)]);
}
#[test]
fn input_24x_yields_integer_and_symbol_tokens() {
let ts = TokenSource::new(std::io::Cursor::new(b"24x"));
let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
assert_eq!(tokens, vec![Token::Integer(24), Token::Symbol('x')]);
}
#[test]
fn input_12_space_34_yields_two_integer_tokens() {
let ts = TokenSource::new(std::io::Cursor::new(b"12 34"));
let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
assert_eq!(tokens, vec![Token::Integer(12), Token::Integer(34)]);
}
#[test]
fn input_y_space_space_12_yields_integer_and_symbol_tokens() {
let ts = TokenSource::new(std::io::Cursor::new(b"y 12"));
let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
assert_eq!(tokens, vec![Token::Symbol('y'), Token::Integer(12)]);
}
#[test]
fn open_paren_close_paren_yield_expected_tokens() {
let ts = TokenSource::new(std::io::Cursor::new(b"()"));
let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
assert_eq!(tokens, vec![Token::OpenParen, Token::CloseParen]);
}
#[test]
fn operators_all_yeild_tokens() {
let ts = TokenSource::new(std::io::Cursor::new(b"+-*/^"));
let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
assert_eq!(
tokens,
vec![
Token::Plus,
Token::Minus,
Token::Times,
Token::Divide,
Token::Caret,
]);
}
#[test]
fn input_ht_cr_lf_yeilds_no_tokens() {
let ts = TokenSource::new(std::io::Cursor::new(b"\t\r\n"));
let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
assert!(tokens.is_empty());
}
}
+496
View File
@@ -0,0 +1,496 @@
use crate::token::Token;
use crate::expression::Expression;
use crate::binding::{Binding, Power};
#[derive(Debug)]
pub enum Error {
Io(std::io::Error),
InvalidSyntax,
}
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Error::Io(err) => write!(f, "I/O Error: {}", err),
Error::InvalidSyntax => write!(f, "Invalid syntax"),
}
}
}
pub fn parse<I>(input: I) -> Result<Expression, Error>
where I: Iterator<Item = Result<Token, std::io::Error>>
{
expr(&mut input.peekable(), Binding::left(Power::Minimum))
}
fn expr<I>(
input: &mut std::iter::Peekable<I>,
limit: Binding) -> Result<Expression, Error>
where I: Iterator<Item = Result<Token, std::io::Error>>
{
let t1 = input.next()
.ok_or(Error::InvalidSyntax)?
.map_err(|err| Error::Io(err))?;
let mut left = nud(input, t1)?;
loop {
match input.peek() {
Some(Ok(op)) => {
if Binding::of_operator(op) > limit {
let op_copy = op.clone();
left = led(input, left, op_copy)?;
} else {
break Ok(left)
}
},
Some(Err(_)) => {
let err = input.next().unwrap().unwrap_err();
break Err(Error::Io(err));
},
None => break Ok(left),
}
}
}
fn nud<I>(
input: &mut std::iter::Peekable<I>,
token: Token) -> Result<Expression, Error>
where I: Iterator<Item = Result<Token, std::io::Error>>
{
match token {
Token::Symbol(c) => Ok(Expression::Symbol(c)),
Token::Integer(x) => Ok(Expression::Integer(x as i32)),
Token::Plus => expr(input, Binding::right(Power::Multiplication)),
Token::Minus => {
let operand =
expr(input, Binding::right(Power::Multiplication))?;
Ok(match operand {
Expression::Integer(x) => Expression::Integer(-x),
_ => Expression::product(Expression::Integer(-1), operand),
})
}
Token::OpenParen => {
let operand = expr(input, Binding::left(Power::Minimum))?;
match input.next() {
Some(Ok(Token::CloseParen)) => Ok(operand),
_ => Err(Error::InvalidSyntax),
}
}
_ => Err(Error::InvalidSyntax),
}
}
fn led<I>(
input: &mut std::iter::Peekable<I>,
left: Expression,
op: Token) -> Result<Expression, Error>
where I: Iterator<Item = Result<Token, std::io::Error>>
{
if op.starts_expression() {
let right = expr(input, Binding::of_operator(&op))?;
Ok(Expression::product(left, right))
} else {
input.next();
let right = expr(input, Binding::of_operator(&op))?;
Ok(match op {
Token::Plus => Expression::sum(left, right),
Token::Minus => Expression::difference(left, right),
Token::Times => Expression::product(left, right),
Token::Divide => Expression::quotient(left, right),
Token::Caret => Expression::exponent(left, right),
_ => unreachable!(),
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn symbol_token_parses_to_symbol() {
let tokens = vec![ Token::Symbol('x') ];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(expr, Expression::Symbol('x'));
}
#[test]
fn integer_token_parses_to_integer() {
let tokens = vec![ Token::Integer(42) ];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(expr, Expression::Integer(42));
}
#[test]
fn x_plus_y_parses_as_sum() {
let tokens = vec![
Token::Symbol('x'),
Token::Plus,
Token::Symbol('y'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::sum(
Expression::Symbol('x'), Expression::Symbol('y')));
}
#[test]
fn x_times_y_parses_as_product() {
let tokens = vec![
Token::Symbol('x'),
Token::Times,
Token::Symbol('y'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::product(
Expression::Symbol('x'), Expression::Symbol('y')));
}
#[test]
fn x_plus_y_times_z_parses_as_sum_of_product() {
let tokens = vec![
Token::Symbol('x'),
Token::Plus,
Token::Symbol('y'),
Token::Times,
Token::Symbol('z'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::sum(
Expression::Symbol('x'),
Expression::product(
Expression::Symbol('y'), Expression::Symbol('z'))));
}
#[test]
fn x_times_y_plus_z_parses_as_sum_of_product() {
let tokens = vec![
Token::Symbol('x'),
Token::Times,
Token::Symbol('y'),
Token::Plus,
Token::Symbol('z'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::sum(
Expression::product(
Expression::Symbol('x'), Expression::Symbol('y')),
Expression::Symbol('z')));
}
#[test]
fn x_caret_y_times_z_parses_as_product_of_exponent() {
let tokens = vec![
Token::Symbol('x'),
Token::Caret,
Token::Symbol('y'),
Token::Times,
Token::Symbol('z'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::product(
Expression::exponent(
Expression::Symbol('x'), Expression::Symbol('y')),
Expression::Symbol('z')));
}
#[test]
fn x_minus_y_minus_z_parses_as_left_associative() {
let tokens = vec![
Token::Symbol('x'),
Token::Minus,
Token::Symbol('y'),
Token::Minus,
Token::Symbol('z'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::difference(
Expression::difference(
Expression::Symbol('x'), Expression::Symbol('y')),
Expression::Symbol('z')));
}
#[test]
fn x_divide_y_divide_z_parses_as_left_associative() {
let tokens = vec![
Token::Symbol('x'),
Token::Divide,
Token::Symbol('y'),
Token::Divide,
Token::Symbol('z'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::quotient(
Expression::quotient(
Expression::Symbol('x'), Expression::Symbol('y')),
Expression::Symbol('z')));
}
#[test]
fn x_caret_y_caret_z_parses_as_right_associative() {
let tokens = vec![
Token::Symbol('x'),
Token::Caret,
Token::Symbol('y'),
Token::Caret,
Token::Symbol('z'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::exponent(
Expression::Symbol('x'),
Expression::exponent(
Expression::Symbol('y'), Expression::Symbol('z')),));
}
#[test]
fn minus_6_parses_as_integer() {
let tokens = vec![ Token::Minus, Token::Integer(6) ];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(expr, Expression::Integer(-6));
}
#[test]
fn minus_x_parses_as_product() {
let tokens = vec![ Token::Minus, Token::Symbol('x') ];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::product(
Expression::Integer(-1), Expression::Symbol('x')));
}
#[test]
fn plus_x_parses_as_symbol() {
let tokens = vec![ Token::Plus, Token::Symbol('x') ];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(expr, Expression::Symbol('x'));
}
#[test]
fn minus_x_times_y_parses_with_higher_precedence_for_unary_minus() {
let tokens = vec![
Token::Minus,
Token::Symbol('x'),
Token::Times,
Token::Symbol('y'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::product(
Expression::product(
Expression::Integer(-1), Expression::Symbol('x')),
Expression::Symbol('y')));
}
#[test]
fn minus_x_caret_y_parses_with_higher_precedence_for_caret() {
let tokens = vec![
Token::Minus,
Token::Symbol('x'),
Token::Caret,
Token::Symbol('y'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::product(
Expression::Integer(-1),
Expression::exponent(
Expression::Symbol('x'), Expression::Symbol('y'))));
}
#[test]
fn x_times_oparen_y_plus_z_cparen_parses_as_product_of_sum() {
let tokens = vec![
Token::Symbol('x'),
Token::Times,
Token::OpenParen,
Token::Symbol('y'),
Token::Plus,
Token::Symbol('z'),
Token::CloseParen,
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::product(
Expression::Symbol('x'),
Expression::sum(
Expression::Symbol('y'), Expression::Symbol('z'))));
}
#[test]
fn x_y_parses_as_product() {
let tokens = vec![
Token::Symbol('x'),
Token::Symbol('y'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::product(
Expression::Symbol('x'), Expression::Symbol('y')));
}
#[test]
fn input_2_x_parses_as_product() {
let tokens = vec![
Token::Integer(2),
Token::Symbol('x'),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::product(
Expression::Integer(2), Expression::Symbol('x')));
}
#[test]
fn x_2_parses_as_product() {
let tokens = vec![
Token::Symbol('x'),
Token::Integer(2),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::product(
Expression::Symbol('x'), Expression::Integer(2)));
}
#[test]
fn complex_expression_1() {
let tokens = vec![
Token::Integer(2),
Token::Symbol('x'),
Token::Caret,
Token::Integer(3),
Token::Minus,
Token::Symbol('y'),
Token::Divide,
Token::Symbol('z'),
Token::Caret,
Token::Integer(2),
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::difference(
Expression::product(
Expression::Integer(2),
Expression::exponent(
Expression::Symbol('x'),
Expression::Integer(3))),
Expression::quotient(
Expression::Symbol('y'),
Expression::exponent(
Expression::Symbol('z'),
Expression::Integer(2)))));
}
#[test]
fn complex_expression_2() {
let tokens = vec![
Token::Symbol('x'),
Token::OpenParen,
Token::Symbol('x'),
Token::Plus,
Token::Symbol('y'),
Token::Minus,
Token::Integer(1),
Token::CloseParen,
Token::Plus,
Token::Symbol('y'),
Token::Caret,
Token::OpenParen,
Token::Integer(3),
Token::Minus,
Token::Symbol('x'),
Token::CloseParen,
];
let expr = parse(tokens.into_iter().map(|t| Ok(t))).unwrap();
assert_eq!(
expr,
Expression::sum(
Expression::product(
Expression::Symbol('x'),
Expression::difference(
Expression::sum(
Expression::Symbol('x'),
Expression::Symbol('y')),
Expression::Integer(1))),
Expression::exponent(
Expression::Symbol('y'),
Expression::difference(
Expression::Integer(3),
Expression::Symbol('x')))));
}
#[test]
fn unclosed_paren_is_invalid_syntax() {
let tokens = vec![
Token::OpenParen,
Token::Symbol('x'),
Token::Plus,
Token::Symbol('y'),
];
let result = parse(tokens.into_iter().map(|t| Ok(t)));
std::assert_matches!(result, Err(Error::InvalidSyntax));
}
#[test]
fn close_paren_x_is_invalid_syntax() {
let tokens = vec![
Token::CloseParen,
Token::Symbol('x'),
];
let result = parse(tokens.into_iter().map(|t| Ok(t)));
std::assert_matches!(result, Err(Error::InvalidSyntax));
}
#[test]
fn times_x_is_invalid_syntax() {
let tokens = vec![
Token::Times,
Token::Symbol('x'),
];
let result = parse(tokens.into_iter().map(|t| Ok(t)));
std::assert_matches!(result, Err(Error::InvalidSyntax));
}
#[test]
fn divide_x_is_invalid_syntax() {
let tokens = vec![
Token::Divide,
Token::Symbol('x'),
];
let result = parse(tokens.into_iter().map(|t| Ok(t)));
std::assert_matches!(result, Err(Error::InvalidSyntax));
}
#[test]
fn caret_x_is_invalid_syntax() {
let tokens = vec![
Token::Caret,
Token::Symbol('x'),
];
let result = parse(tokens.into_iter().map(|t| Ok(t)));
std::assert_matches!(result, Err(Error::InvalidSyntax));
}
}