Implement tokenisation

lib/include/expr.h

@@ -0,0 +1,13 @@
+#ifndef EXPR_H
+#define EXPR_H
+
+#define MAX_SYMBOL_LEN 32U
+
+#include <stddef.h>
+
+typedef struct {
+	char buf[MAX_SYMBOL_LEN];
+	size_t len;
+} symbol_t;
+
+#endif

lib/include/memory_stream.h

@@ -0,0 +1,15 @@
+#ifndef MEMORY_STREAM_H
+#define MEMORY_STREAM_H
+
+#include "stream.h"
+
+#include <stddef.h>
+
+typedef struct {
+	stream_t stream;
+	const uint8_t *src, *end;
+} memory_stream_t;
+
+void memory_stream_init(memory_stream_t *s, const uint8_t *src, size_t size);
+
+#endif

lib/include/stream.h

@@ -0,0 +1,20 @@
+#ifndef STREAM_H
+#define STREAM_H
+
+#include <stdint.h>
+
+#define STREAM_GET_BYTE(stream, out) stream->get_byte(stream, out)
+#define STREAM_PEEK_BYTE(stream, out) stream->peek_byte(stream, out)
+
+typedef enum {
+	STREAM_STATUS_OK,
+	STREAM_STATUS_ERROR,
+	STREAM_STATUS_END,
+} stream_status_t;
+
+typedef struct stream {
+	stream_status_t (*get_byte)(struct stream *s, uint8_t *out);
+	stream_status_t (*peek_byte)(struct stream *s, uint8_t *out);
+} stream_t;
+
+#endif

lib/include/token.h

@@ -0,0 +1,26 @@
+#ifndef TOKEN_H
+#define TOKEN_H
+
+#include "expr.h"
+#include "stream.h"
+
+typedef enum {
+	TOKEN_TYPE_INTEGER,
+	TOKEN_TYPE_SYMBOL,
+	TOKEN_TYPE_OPEN_PAREN,
+	TOKEN_TYPE_CLOSE_PAREN,
+} token_type_t;
+
+typedef struct {
+	token_type_t type;
+	union {
+		int64_t integer;
+		symbol_t symbol;
+	};
+} token_t;
+
+typedef enum { TOKEN_OK, TOKEN_FAILED } token_status_t;
+
+token_status_t token_read(stream_t *input, token_t *out);
+
+#endif