cdo/wipforth
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework tokenizer to avoid unecessary allocations

Browse Source
This commit is contained in:
Camden Dixie O'Brien
2026-03-20 12:47:33 +00:00
parent c20e7e181b
commit 36429bf8bc
1 changed files with 25 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
asm.js
View File

@@ -14,20 +14,29 @@ class Tokenizer {
this.buffer = []; this.buffer = [];
this.comment = false; this.comment = false;
this.string = false; this.string = false;
this.cursor = 0;
} }
find(pred) {
for (let i = this.cursor; i < this.buffer.length; ++i) {
if (pred(this.buffer[i]))
return i;
}
return null;
}
skip() { skip() {
const idx = this.buffer.findIndex((cp) => !this.skips.has(cp)); const idx = this.find((cp) => !this.skips.has(cp));
this.buffer = idx == -1 ? [] : this.buffer.slice(idx); this.cursor = idx == null ? this.buffer.length : idx;
} }
next_string() { next_string() {
const idx = this.buffer.findIndex((cp) => cp == this.string_quote); const idx = this.find((cp) => cp == this.string_quote);
if (idx == -1) { if (idx == null) {
this.string = true; this.string = true;
} else { } else {
const string = this.buffer.slice(0, idx).join(""); const string = this.buffer.slice(this.cursor, idx).join("");
this.buffer = this.buffer.slice(idx + 1); this.cursor = idx + 1;
this.string = false; this.string = false;
return { string: string }; return { string: string };
} }
@@ -38,18 +47,20 @@ class Tokenizer {
return this.next_string(); return this.next_string();
this.skip(); this.skip();
if (this.buffer[0] == LINE_END) if (this.buffer[this.cursor] == LINE_END) {
return this.buffer.shift(); ++this.cursor;
return LINE_END;
}
if (this.buffer[0] == this.string_quote) { if (this.buffer[this.cursor] == this.string_quote) {
this.buffer.shift(); ++this.cursor;
return this.next_string(); return this.next_string();
} }
const idx = this.buffer.findIndex((cp) => this.delims.has(cp)); const idx = this.find((cp) => this.delims.has(cp));
if (idx != -1) { if (idx != null) {
const token = this.buffer.slice(0, idx).join(""); const token = this.buffer.slice(this.cursor, idx).join("");
this.buffer = this.buffer.slice(idx); this.cursor = idx;
return token; return token;
} }
} }