diff --git a/src/lex.rs b/src/lex.rs new file mode 100644 index 0000000..722ae3d --- /dev/null +++ b/src/lex.rs @@ -0,0 +1,182 @@ +use crate::token::Token; + +#[derive(Debug)] +enum State { + Init, + Integer(i32), +} + +pub struct TokenSource { + input: R, + state: State, +} + +const RADIX: u32 = 10; + +fn skip(c: char) -> bool { + return c.is_whitespace(); +} + +fn token_of(c: char) -> Token { + match c { + '(' => Token::OpenParen, + ')' => Token::CloseParen, + '+' => Token::Plus, + '-' => Token::Minus, + '*' => Token::Times, + '/' => Token::Divide, + '^' => Token::Caret, + _ => Token::Symbol(c), + } +} + +impl TokenSource { + pub fn new(input: R) -> Self { + Self { input, state: State::Init } + } + + fn handle_char_init(&mut self, c: char) -> Option { + self.input.consume(1); + if skip(c) { + None + } else if let Some(digit) = c.to_digit(RADIX) { + self.state = State::Integer(digit as i32); + None + } else { + Some(token_of(c)) + } + } + + fn handle_char_integer(&mut self, value: i32, c: char) -> Option { + if let Some(digit) = c.to_digit(RADIX) { + self.input.consume(1); + self.state = State::Integer( + (RADIX as i32) * value + digit as i32); + None + } else { + self.state = State::Init; + Some(Token::Integer(value)) + } + } + + fn dispatch_byte(&mut self, byte: u8) -> Option { + let c = byte as char; + match self.state { + State::Init => self.handle_char_init(c), + State::Integer(value) => self.handle_char_integer(value, c), + } + } + + fn dispatch_eof(&mut self) -> Option { + match self.state { + State::Init => None, + State::Integer(value) => { + self.state = State::Init; + Some(Token::Integer(value)) + } + } + } +} + +impl Iterator for TokenSource { + type Item = Result; + + fn next(&mut self) -> Option { + loop { + match self.input.fill_buf() { + Ok(buf) => match buf.first() { + Some(&byte) => { + if let Some(token) = self.dispatch_byte(byte) { + break Some(Ok(token)); + } + } + None => break self.dispatch_eof().map(|token| Ok(token)) + } + Err(err) => break Some(Err(err)) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_input_yields_no_tokens() { + let ts = TokenSource::new(std::io::Cursor::new(b"")); + let tokens: Vec = ts.collect::>().unwrap(); + assert!(tokens.is_empty()); + } + + #[test] + fn input_x_yields_one_symbol_token() { + let ts = TokenSource::new(std::io::Cursor::new(b"x")); + let tokens: Vec = ts.collect::>().unwrap(); + assert_eq!(tokens, vec![Token::Symbol('x')]); + } + + #[test] + fn input_xy_yeilds_two_symbol_tokens() { + let ts = TokenSource::new(std::io::Cursor::new(b"xy")); + let tokens: Vec = ts.collect::>().unwrap(); + assert_eq!(tokens, vec![Token::Symbol('x'), Token::Symbol('y')]); + } + + #[test] + fn input_12_yields_one_integer_token() { + let ts = TokenSource::new(std::io::Cursor::new(b"12")); + let tokens: Vec = ts.collect::>().unwrap(); + assert_eq!(tokens, vec![Token::Integer(12)]); + } + + #[test] + fn input_24x_yields_integer_and_symbol_tokens() { + let ts = TokenSource::new(std::io::Cursor::new(b"24x")); + let tokens: Vec = ts.collect::>().unwrap(); + assert_eq!(tokens, vec![Token::Integer(24), Token::Symbol('x')]); + } + + #[test] + fn input_12_space_34_yields_two_integer_tokens() { + let ts = TokenSource::new(std::io::Cursor::new(b"12 34")); + let tokens: Vec = ts.collect::>().unwrap(); + assert_eq!(tokens, vec![Token::Integer(12), Token::Integer(34)]); + } + + #[test] + fn input_y_space_space_12_yields_integer_and_symbol_tokens() { + let ts = TokenSource::new(std::io::Cursor::new(b"y 12")); + let tokens: Vec = ts.collect::>().unwrap(); + assert_eq!(tokens, vec![Token::Symbol('y'), Token::Integer(12)]); + } + + #[test] + fn open_paren_close_paren_yield_expected_tokens() { + let ts = TokenSource::new(std::io::Cursor::new(b"()")); + let tokens: Vec = ts.collect::>().unwrap(); + assert_eq!(tokens, vec![Token::OpenParen, Token::CloseParen]); + } + + #[test] + fn operators_all_yeild_tokens() { + let ts = TokenSource::new(std::io::Cursor::new(b"+-*/^")); + let tokens: Vec = ts.collect::>().unwrap(); + assert_eq!( + tokens, + vec![ + Token::Plus, + Token::Minus, + Token::Times, + Token::Divide, + Token::Caret, + ]); + } + + #[test] + fn input_ht_cr_lf_yeilds_no_tokens() { + let ts = TokenSource::new(std::io::Cursor::new(b"\t\r\n")); + let tokens: Vec = ts.collect::>().unwrap(); + assert!(tokens.is_empty()); + } +} diff --git a/src/lib.rs b/src/lib.rs index 79c66ba..f793d1a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1 +1,2 @@ pub mod token; +pub mod lex;