asm.js

@@ -9,9 +9,11 @@ class Tokenizer {
 	constructor() {
 		this.delims = new Set([" ", "\r", "\n", "\t"]);
 		this.skips = new Set([" ", "\r", "\t"]);
-        this.comment_start = ";"
+        this.comment_start = ";";
+        this.string_quote = '"';
 		this.buffer = [];
         this.comment = false;
+        this.string = false;
 	}
 
 	skip() {
@@ -19,11 +21,31 @@ class Tokenizer {
         this.buffer = idx == -1 ? [] : this.buffer.slice(idx);
     }
 
+    next_string() {
+        const idx = this.buffer.findIndex((cp) => cp == this.string_quote);
+        if (idx == -1) {
+            this.string = true;
+        } else {
+            const string = this.buffer.slice(0, idx).join("");
+            this.buffer = this.buffer.slice(idx + 1);
+            this.string = false;
+            return { string: string };
+        }
+    }
+
 	next() {
+        if (this.string)
+            return this.next_string();
+
         this.skip();
         if (this.buffer[0] == LINE_END)
             return this.buffer.shift();
 
+        if (this.buffer[0] == this.string_quote) {
+            this.buffer.shift();
+            return this.next_string();
+        }
+
 		const idx = this.buffer.findIndex((cp) => this.delims.has(cp));
         if (idx != -1) {
             const token = this.buffer.slice(0, idx).join("");