Implement tokenisation

lib/include/token.h

@@ -0,0 +1,26 @@
+#ifndef TOKEN_H
+#define TOKEN_H
+
+#include "expr.h"
+#include "stream.h"
+
+typedef enum {
+	TOKEN_TYPE_INTEGER,
+	TOKEN_TYPE_SYMBOL,
+	TOKEN_TYPE_OPEN_PAREN,
+	TOKEN_TYPE_CLOSE_PAREN,
+} token_type_t;
+
+typedef struct {
+	token_type_t type;
+	union {
+		int64_t integer;
+		symbol_t symbol;
+	};
+} token_t;
+
+typedef enum { TOKEN_OK, TOKEN_FAILED } token_status_t;
+
+token_status_t token_read(stream_t *input, token_t *out);
+
+#endif