Implement tokenisation

lib/include/memory_stream.h

@@ -0,0 +1,15 @@
+#ifndef MEMORY_STREAM_H
+#define MEMORY_STREAM_H
+
+#include "stream.h"
+
+#include <stddef.h>
+
+typedef struct {
+	stream_t stream;
+	const uint8_t *src, *end;
+} memory_stream_t;
+
+void memory_stream_init(memory_stream_t *s, const uint8_t *src, size_t size);
+
+#endif