Add string support to tokenizer

This commit is contained in:
2026-03-10 17:53:58 +00:00
parent 5a3084dd16
commit 672a453f6c

24
asm.js
View File

@@ -9,9 +9,11 @@ class Tokenizer {
constructor() {
this.delims = new Set([" ", "\r", "\n", "\t"]);
this.skips = new Set([" ", "\r", "\t"]);
this.comment_start = ";"
this.comment_start = ";";
this.string_quote = '"';
this.buffer = [];
this.comment = false;
this.string = false;
}
skip() {
@@ -19,11 +21,31 @@ class Tokenizer {
this.buffer = idx == -1 ? [] : this.buffer.slice(idx);
}
next_string() {
const idx = this.buffer.findIndex((cp) => cp == this.string_quote);
if (idx == -1) {
this.string = true;
} else {
const string = this.buffer.slice(0, idx).join("");
this.buffer = this.buffer.slice(idx + 1);
this.string = false;
return { string: string };
}
}
next() {
if (this.string)
return this.next_string();
this.skip();
if (this.buffer[0] == LINE_END)
return this.buffer.shift();
if (this.buffer[0] == this.string_quote) {
this.buffer.shift();
return this.next_string();
}
const idx = this.buffer.findIndex((cp) => this.delims.has(cp));
if (idx != -1) {
const token = this.buffer.slice(0, idx).join("");