Rework tokenizer to avoid unecessary allocations

asm.js

@@ -14,20 +14,29 @@ class Tokenizer {
 		this.buffer = [];
         this.comment = false;
         this.string = false;
+        this.cursor = 0;
 	}
 
+    find(pred) {
+        for (let i = this.cursor; i < this.buffer.length; ++i) {
+            if (pred(this.buffer[i]))
+                return i;
+        }
+        return null;
+    }
+
 	skip() {
-		const idx = this.buffer.findIndex((cp) => !this.skips.has(cp));
-        this.buffer = idx == -1 ? [] : this.buffer.slice(idx);
+		const idx = this.find((cp) => !this.skips.has(cp));
+        this.cursor = idx == null ? this.buffer.length : idx;
     }
 
     next_string() {
-        const idx = this.buffer.findIndex((cp) => cp == this.string_quote);
-        if (idx == -1) {
+        const idx = this.find((cp) => cp == this.string_quote);
+        if (idx == null) {
             this.string = true;
         } else {
-            const string = this.buffer.slice(0, idx).join("");
-            this.buffer = this.buffer.slice(idx + 1);
+            const string = this.buffer.slice(this.cursor, idx).join("");
+            this.cursor = idx + 1;
             this.string = false;
             return { string: string };
         }
@@ -38,18 +47,20 @@ class Tokenizer {
             return this.next_string();
 
         this.skip();
-        if (this.buffer[0] == LINE_END)
-            return this.buffer.shift();
+        if (this.buffer[this.cursor] == LINE_END) {
+            ++this.cursor;
+            return LINE_END;
+        }
 
-        if (this.buffer[0] == this.string_quote) {
-            this.buffer.shift();
+        if (this.buffer[this.cursor] == this.string_quote) {
+            ++this.cursor;
             return this.next_string();
         }
 
-		const idx = this.buffer.findIndex((cp) => this.delims.has(cp));
-        if (idx != -1) {
-            const token = this.buffer.slice(0, idx).join("");
-            this.buffer = this.buffer.slice(idx);
+		const idx = this.find((cp) => this.delims.has(cp));
+        if (idx != null) {
+            const token = this.buffer.slice(this.cursor, idx).join("");
+            this.cursor = idx;
             return token;
         }
 	}