Implement lexing

src/lex.rs

@@ -0,0 +1,182 @@
+use crate::token::Token;
+
+#[derive(Debug)]
+enum State {
+	Init,
+	Integer(i32),
+}
+
+pub struct TokenSource<R> {
+	input: R,
+	state: State,
+}
+
+const RADIX: u32 = 10;
+
+fn skip(c: char) -> bool {
+	return c.is_whitespace();
+}
+
+fn token_of(c: char) -> Token {
+	match c {
+		'(' => Token::OpenParen,
+		')' => Token::CloseParen,
+		'+' => Token::Plus,
+		'-' => Token::Minus,
+		'*' => Token::Times,
+		'/' => Token::Divide,
+		'^' => Token::Caret,
+		_ => Token::Symbol(c),
+	}
+}
+
+impl<R: std::io::BufRead> TokenSource<R> {
+	pub fn new(input: R) -> Self {
+		Self { input, state: State::Init }
+	}
+
+	fn handle_char_init(&mut self, c: char) -> Option<Token> {
+		self.input.consume(1);
+		if skip(c) {
+			None
+		} else if let Some(digit) = c.to_digit(RADIX) {
+			self.state = State::Integer(digit as i32);
+			None
+		} else {
+			Some(token_of(c))
+		}
+	}
+
+	fn handle_char_integer(&mut self, value: i32, c: char) -> Option<Token> {
+		if let Some(digit) = c.to_digit(RADIX) {
+			self.input.consume(1);
+			self.state = State::Integer(
+				(RADIX as i32) * value + digit as i32);
+			None
+		} else {
+			self.state = State::Init;
+			Some(Token::Integer(value))
+		}
+	}
+
+	fn dispatch_byte(&mut self, byte: u8) -> Option<Token> {
+		let c = byte as char;
+		match self.state {
+			State::Init => self.handle_char_init(c),
+			State::Integer(value) => self.handle_char_integer(value, c),
+		}
+	}
+
+	fn dispatch_eof(&mut self) -> Option<Token> {
+		match self.state {
+			State::Init => None,
+			State::Integer(value) => {
+				self.state = State::Init;
+				Some(Token::Integer(value))
+			}
+		}
+	}
+}
+
+impl<R: std::io::BufRead> Iterator for TokenSource<R> {
+	type Item = Result<Token, std::io::Error>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		loop {
+			match self.input.fill_buf() {
+				Ok(buf) => match buf.first() {
+					Some(&byte) => {
+						if let Some(token) = self.dispatch_byte(byte) {
+							break Some(Ok(token));
+						}
+					}
+					None => break self.dispatch_eof().map(|token| Ok(token))
+				}
+				Err(err) => break Some(Err(err))
+			}
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn empty_input_yields_no_tokens() {
+		let ts = TokenSource::new(std::io::Cursor::new(b""));
+		let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
+		assert!(tokens.is_empty());
+    }
+
+	#[test]
+	fn input_x_yields_one_symbol_token() {
+		let ts = TokenSource::new(std::io::Cursor::new(b"x"));
+		let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
+		assert_eq!(tokens, vec![Token::Symbol('x')]);
+	}
+
+	#[test]
+	fn input_xy_yeilds_two_symbol_tokens() {
+		let ts = TokenSource::new(std::io::Cursor::new(b"xy"));
+		let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
+		assert_eq!(tokens, vec![Token::Symbol('x'), Token::Symbol('y')]);
+	}
+
+	#[test]
+	fn input_12_yields_one_integer_token() {
+		let ts = TokenSource::new(std::io::Cursor::new(b"12"));
+		let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
+		assert_eq!(tokens, vec![Token::Integer(12)]);
+	}
+
+	#[test]
+	fn input_24x_yields_integer_and_symbol_tokens() {
+		let ts = TokenSource::new(std::io::Cursor::new(b"24x"));
+		let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
+		assert_eq!(tokens, vec![Token::Integer(24), Token::Symbol('x')]);
+	}
+
+	#[test]
+	fn input_12_space_34_yields_two_integer_tokens() {
+		let ts = TokenSource::new(std::io::Cursor::new(b"12 34"));
+		let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
+		assert_eq!(tokens, vec![Token::Integer(12), Token::Integer(34)]);
+	}
+
+	#[test]
+	fn input_y_space_space_12_yields_integer_and_symbol_tokens() {
+		let ts = TokenSource::new(std::io::Cursor::new(b"y  12"));
+		let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
+		assert_eq!(tokens, vec![Token::Symbol('y'), Token::Integer(12)]);
+	}
+
+	#[test]
+	fn open_paren_close_paren_yield_expected_tokens() {
+		let ts = TokenSource::new(std::io::Cursor::new(b"()"));
+		let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
+		assert_eq!(tokens, vec![Token::OpenParen, Token::CloseParen]);
+	}
+
+	#[test]
+	fn operators_all_yeild_tokens() {
+		let ts = TokenSource::new(std::io::Cursor::new(b"+-*/^"));
+		let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
+		assert_eq!(
+			tokens,
+			vec![
+				Token::Plus,
+				Token::Minus,
+				Token::Times,
+				Token::Divide,
+				Token::Caret,
+			]);
+	}
+
+	#[test]
+	fn input_ht_cr_lf_yeilds_no_tokens() {
+		let ts = TokenSource::new(std::io::Cursor::new(b"\t\r\n"));
+		let tokens: Vec<Token> = ts.collect::<Result<_, _>>().unwrap();
+		assert!(tokens.is_empty());
+	}
+}

src/lib.rs

@@ -1 +1,2 @@
 pub mod token;
+pub mod lex;