cdo/imp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement tokenisation

Browse Source
This commit is contained in:
Camden Dixie O'Brien
2025-08-09 12:13:55 +01:00
parent 657f0922bb
commit 3e8a9d6789
10 changed files with 550 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
tests/CMakeLists.txt Normal file
View File

@@ -0,0 +1,13 @@
function(add_test_suites)
foreach (source ${ARGN})
string(REGEX REPLACE ".c$" "" name ${source})
add_executable(${name} ${source})
configure_target(${name})
target_link_libraries(${name} PRIVATE imp unity)
add_test(NAME ${name} COMMAND ${name})
endforeach()
endfunction()
add_test_suites(
token_tests.c
)

329
tests/token_tests.c Normal file
View File

@@ -0,0 +1,329 @@
#include "memory_stream.h"
#include "token.h"
#include "unity.h"
#include <string.h>
void setUp(void)
{
}
void tearDown(void)
{
}
static void test_123(void)
{
const char *input = "123";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_INTEGER, token.type);
TEST_ASSERT_EQUAL(123, token.integer);
}
static void test_321(void)
{
const char *input = "321";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_INTEGER, token.type);
TEST_ASSERT_EQUAL(321, token.integer);
}
static void test_foo(void)
{
const char *input = "foo";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_SYMBOL, token.type);
TEST_ASSERT_EQUAL(3, token.symbol.len);
TEST_ASSERT_EQUAL_MEMORY("foo", token.symbol.buf, 3);
}
static void test_quux(void)
{
const char *input = "quux";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_SYMBOL, token.type);
TEST_ASSERT_EQUAL(4, token.symbol.len);
TEST_ASSERT_EQUAL_MEMORY("quux", token.symbol.buf, 3);
}
static void test_space_space_space_456(void)
{
const char *input = " 456";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_INTEGER, token.type);
TEST_ASSERT_EQUAL(456, token.integer);
}
static void test_tab_tab_bar(void)
{
const char *input = "\t\tbar";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_SYMBOL, token.type);
TEST_ASSERT_EQUAL(3, token.symbol.len);
TEST_ASSERT_EQUAL_MEMORY("bar", token.symbol.buf, 3);
}
static void test_12_space_34(void)
{
const char *input = "12 34";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_INTEGER, token.type);
TEST_ASSERT_EQUAL(12, token.integer);
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_INTEGER, token.type);
TEST_ASSERT_EQUAL(34, token.integer);
}
static void test_12_tab_34(void)
{
const char *input = "12\t34";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_INTEGER, token.type);
TEST_ASSERT_EQUAL(12, token.integer);
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_INTEGER, token.type);
TEST_ASSERT_EQUAL(34, token.integer);
}
static void test_foo_space_bar(void)
{
const char *input = "foo bar";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_SYMBOL, token.type);
TEST_ASSERT_EQUAL(3, token.symbol.len);
TEST_ASSERT_EQUAL_MEMORY("foo", token.symbol.buf, 3);
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_SYMBOL, token.type);
TEST_ASSERT_EQUAL(3, token.symbol.len);
TEST_ASSERT_EQUAL_MEMORY("bar", token.symbol.buf, 3);
}
static void test_foo_tab_bar(void)
{
const char *input = "foo\tbar";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_SYMBOL, token.type);
TEST_ASSERT_EQUAL(3, token.symbol.len);
TEST_ASSERT_EQUAL_MEMORY("foo", token.symbol.buf, 3);
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_SYMBOL, token.type);
TEST_ASSERT_EQUAL(3, token.symbol.len);
TEST_ASSERT_EQUAL_MEMORY("bar", token.symbol.buf, 3);
}
static void test_open_paren(void)
{
const char *input = "(";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_OPEN_PAREN, token.type);
}
static void test_close_paren(void)
{
const char *input = ")";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_CLOSE_PAREN, token.type);
}
static void test_42_open_paren(void)
{
const char *input = "42(";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_INTEGER, token.type);
TEST_ASSERT_EQUAL(42, token.integer);
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_OPEN_PAREN, token.type);
}
static void test_42_close_paren(void)
{
const char *input = "42)";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_INTEGER, token.type);
TEST_ASSERT_EQUAL(42, token.integer);
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_CLOSE_PAREN, token.type);
}
static void test_open_paren_foo_close_paren(void)
{
const char *input = "(foo)";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_OPEN_PAREN, token.type);
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_SYMBOL, token.type);
TEST_ASSERT_EQUAL(3, token.symbol.len);
TEST_ASSERT_EQUAL_MEMORY("foo", token.symbol.buf, 3);
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_OK, status);
TEST_ASSERT_EQUAL(TOKEN_TYPE_CLOSE_PAREN, token.type);
}
static void test_empty_input(void)
{
const char *input = "";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_FAILED, status);
}
static void test_space_tab(void)
{
const char *input = " \t";
memory_stream_t stream;
memory_stream_init(&stream, (const uint8_t *)input, strlen(input));
token_t token;
token_status_t status;
status = token_read((stream_t *)&stream, &token);
TEST_ASSERT_EQUAL(TOKEN_FAILED, status);
}
int main(void)
{
UNITY_BEGIN();
RUN_TEST(test_123);
RUN_TEST(test_321);
RUN_TEST(test_foo);
RUN_TEST(test_quux);
RUN_TEST(test_space_space_space_456);
RUN_TEST(test_tab_tab_bar);
RUN_TEST(test_12_space_34);
RUN_TEST(test_12_tab_34);
RUN_TEST(test_foo_space_bar);
RUN_TEST(test_foo_tab_bar);
RUN_TEST(test_open_paren);
RUN_TEST(test_close_paren);
RUN_TEST(test_42_open_paren);
RUN_TEST(test_42_close_paren);
RUN_TEST(test_open_paren_foo_close_paren);
RUN_TEST(test_empty_input);
RUN_TEST(test_space_tab);
return UNITY_END();
}