Implement tokenisation

lib/token.c

@@ -0,0 +1,71 @@
+#include "token.h"
+
+#include <assert.h>
+#include <ctype.h>
+#include <stdbool.h>
+
+typedef enum {
+	STATE_INIT,
+	STATE_INTEGER,
+	STATE_SYMBOL,
+	STATE_FINISHED,
+} state_t;
+
+static bool is_delim(uint8_t byte)
+{
+	return isspace(byte) || byte == '(' || byte == ')';
+}
+
+token_status_t token_read(stream_t *input, token_t *out)
+{
+	state_t state = STATE_INIT;
+	uint8_t byte;
+	stream_status_t status;
+
+	while (state != STATE_FINISHED) {
+		status = STREAM_PEEK_BYTE(input, &byte);
+		if (status != STREAM_STATUS_OK
+		    || (state != STATE_INIT && is_delim(byte)))
+			break;
+
+		status = STREAM_GET_BYTE(input, &byte);
+		if (status != STREAM_STATUS_OK)
+			break;
+		switch (state) {
+		case STATE_INIT:
+			if (byte == '(') {
+				out->type = TOKEN_TYPE_OPEN_PAREN;
+				state = STATE_FINISHED;
+			} else if (byte == ')') {
+				out->type = TOKEN_TYPE_CLOSE_PAREN;
+				state = STATE_FINISHED;
+			} else if (isdigit(byte)) {
+				out->type = TOKEN_TYPE_INTEGER;
+				out->integer = byte - '0';
+				state = STATE_INTEGER;
+			} else if (!isspace(byte)) {
+				out->type = TOKEN_TYPE_SYMBOL;
+				out->symbol.buf[0] = byte;
+				out->symbol.len = 1;
+				state = STATE_SYMBOL;
+			}
+			break;
+
+		case STATE_INTEGER:
+			assert(isdigit(byte));
+			out->integer *= 10;
+			out->integer += byte - '0';
+			break;
+
+		case STATE_SYMBOL:
+			assert(out->symbol.len < MAX_SYMBOL_LEN);
+			out->symbol.buf[out->symbol.len++] = byte;
+			break;
+
+		case STATE_FINISHED:
+			break;
+		}
+	}
+
+	return state != STATE_INIT ? TOKEN_OK : TOKEN_FAILED;
+}