Separate ZIO and MBuffer data structures

  - lua_load no longer takes a null reader function

6 files changed, 123 insertions, 131 deletions

diff --git a/src/lapi.js b/src/lapi.js
index 8897fb2..885dd8f 100644
--- a/src/lapi.js
+++ b/src/lapi.js
@@ -7,7 +7,6 @@ const ldebug    = require('./ldebug.js');
 const ldo       = require('./ldo.js');
 const ldump     = require('./ldump.js');
 const lfunc     = require('./lfunc.js');
-const llex      = require('./llex.js');
 const lobject   = require('./lobject.js');
 const lstate    = require('./lstate.js');
 const lstring   = require('./lstring.js');
@@ -15,6 +14,7 @@ const ltm       = require('./ltm.js');
 const luaconf   = require('./luaconf.js');
 const lvm       = require('./lvm.js');
 const ltable    = require('./ltable.js');
+const lzio      = require('./lzio.js');
 const MAXUPVAL  = lfunc.MAXUPVAL;
 const CT        = defs.constant_types;
 const TS        = defs.thread_status;
@@ -891,8 +891,8 @@ const lua_arith = function(L, op) {
 const lua_load = function(L, reader, data, chunckname, mode) {
     assert(Array.isArray(chunckname), "lua_load expect an array of byte as chunckname");
     assert(mode ? Array.isArray(mode) : true, "lua_load expect an array of byte as mode");
-    let z = new llex.MBuffer(L, data, reader);
     if (!chunckname) chunckname = [defs.char["?"]];
+    let z = new lzio.ZIO(L, reader, data);
     let status = ldo.luaD_protectedparser(L, z, chunckname, mode);
     if (status === TS.LUA_OK) {  /* no errors? */
         let f = L.stack[L.top - 1].value; /* get newly created function */
diff --git a/src/lauxlib.js b/src/lauxlib.js
index 3019bf3..c5a3b21 100644
--- a/src/lauxlib.js
+++ b/src/lauxlib.js
@@ -691,18 +691,9 @@ if (!WEB) {
             this.f = null;  /* file being read */
             this.buff = new Buffer(1024);  /* area for reading file */
             this.pos = 0;  /* current position in file */
-            this.binary = false;
         }
     }
 
-    const toDataView = function(buffer, bytes) {
-        let ab = new ArrayBuffer(bytes);
-        let au = new Uint8Array(ab);
-        for (let i = 0; i < bytes; i++)
-            au[i] = buffer[i];
-        return new DataView(ab);
-    };
-
     const getF = function(L, ud) {
         let lf = ud;
         let bytes = 0;
@@ -715,7 +706,7 @@ if (!WEB) {
             lf.pos += bytes;
         }
         if (bytes > 0)
-            return lf.binary ? toDataView(lf.buff, bytes) : lf.buff.slice(0, bytes);
+            return lf.buff.slice(0, bytes); /* slice on a node.js Buffer is 'free' */
         else return null;
     };
 
@@ -794,7 +785,7 @@ if (!WEB) {
             let com = skipcomment(lf);
             /* check for signature first, as we don't want to add line number corrections in binary case */
             if (com.c === lua.LUA_SIGNATURE.charCodeAt(0) && filename) {  /* binary file? */
-                lf.binary = true;
+                /* no need to re-open in node.js */
             } else if (com.skipped) { /* read initial portion */
                 lf.buff[lf.n++] = '\n'.charCodeAt(0);  /* add line to correct line numbers */
             }
diff --git a/src/ldo.js b/src/ldo.js
index 5564e27..bb34216 100644
--- a/src/ldo.js
+++ b/src/ldo.js
@@ -7,7 +7,6 @@ const defs    = require('./defs.js');
 const lapi    = require('./lapi.js');
 const ldebug  = require('./ldebug.js');
 const lfunc   = require('./lfunc.js');
-const llex    = require('./llex.js');
 const llimit  = require('./llimit.js');
 const lobject = require('./lobject.js');
 const lparser = require('./lparser.js');
@@ -16,6 +15,7 @@ const lstring = require('./lstring.js');
 const ltm     = require('./ltm.js');
 const lundump = require('./lundump.js');
 const lvm     = require('./lvm.js');
+const lzio    = require('./lzio.js');
 
 const CT = defs.constant_types;
 const TS = defs.thread_status;
@@ -527,12 +527,12 @@ const luaD_callnoyield = function(L, off, nResults) {
 ** Execute a protected parser.
 */
 class SParser {
-    constructor() {  /* data to 'f_parser' */
-        this.z = new llex.MBuffer();
-        this.buff = new llex.MBuffer();  /* dynamic structure used by the scanner */
+    constructor(z, name, mode) {  /* data to 'f_parser' */
+        this.z = z;
+        this.buff = new lzio.MBuffer();  /* dynamic structure used by the scanner */
         this.dyd = new lparser.Dyndata();  /* dynamic structures used by the parser */
-        this.mode = null;
-        this.name = null;
+        this.mode = mode;
+        this.name = name;
     }
 }
 
@@ -545,7 +545,7 @@ const checkmode = function(L, mode, x) {
 
 const f_parser = function(L, p) {
     let cl;
-    let c = p.z.getc();  /* read first character */
+    let c = p.z.zgetc();  /* read first character */
     if (c === defs.LUA_SIGNATURE.charCodeAt(0)) {
         checkmode(L, p.mode, defs.to_luastring("binary", true));
         cl = lundump.luaU_undump(L, p.z, p.name);
@@ -559,24 +559,10 @@ const f_parser = function(L, p) {
 };
 
 const luaD_protectedparser = function(L, z, name, mode) {
-    let p = new SParser();
+    let p = new SParser(z, name, mode);
     L.nny++;  /* cannot yield during parsing */
-
-    p.z = z;
-    p.buff.L = L;
-    p.name = name;
-    p.mode = mode;
-    p.dyd.actvar.arr = [];
-    p.dyd.actvar.size = 0;
-    p.dyd.gt.arr = [];
-    p.dyd.gt.size = 0;
-    p.dyd.label.arr = [];
-    p.dyd.label.size = 0;
-
     let status = luaD_pcall(L, f_parser, p, L.top, L.errfunc);
-
     L.nny--;
-
     return status;
 };
 
diff --git a/src/llex.js b/src/llex.js
index 0825146..41b3ec8 100644
--- a/src/llex.js
+++ b/src/llex.js
@@ -69,63 +69,6 @@ const luaX_tokens = [
     "<number>", "<integer>", "<name>", "<string>"
 ];
 
-class MBuffer {
-    constructor(L, data, reader) {
-        this.L = L;
-        this.data = data;
-        this.n = 0;
-        this.buffer = null;
-        this.off = 0;
-        this.reader = reader ? reader : null;
-
-        if (!this.reader) {
-            assert(typeof data !== "string", "Should only load binary of array of bytes");
-            this.buffer = data ? data : [];
-            this.n = this.buffer instanceof DataView ? this.buffer.byteLength : this.buffer.length;
-            this.off = 0;
-        }
-    }
-
-    getc() {
-        if (this.n <= 0)
-            this.fill();
-        if (this.n <= 0)
-            return -1;
-        let r;
-        if (this.buffer instanceof DataView) {
-            r = this.buffer.getUint8(this.off++, true);
-        } else {
-            r = this.buffer[this.off++];
-        }
-        if (this.n-- === 0) // remove reference to input so it can get freed
-            this.buffer = null;
-        return r;
-    }
-
-    read(size) {
-        let r = [];
-
-        while (size > 0) {
-            let byte = this.getc();
-            if (byte !== -1) r.push(byte);
-            size--;
-        }
-
-        return r;
-    }
-
-    fill() {
-        if (this.reader) {
-            this.buffer = this.reader(this.L, this.data);
-            assert(typeof this.buffer !== "string", "Should only load binary of array of bytes");
-            if (this.buffer !== null) {
-                this.n = ((this.buffer instanceof DataView) ? this.buffer.byteLength : this.buffer.length);
-                this.off = 0;
-            }
-        }
-    }
-}
-
 class SemInfo {
     constructor() {
         this.r = NaN;
@@ -152,8 +95,8 @@ class LexState {
         this.lookahead = new Token();  /* look ahead token */
         this.fs = null;  /* current function (parser) */
         this.L = null;
-        this.z = new MBuffer();
-        this.buff = new MBuffer();  /* buffer for tokens */
+        this.z = null; /* input stream */
+        this.buff = null;  /* buffer for tokens */
         this.h = null;  /* to reuse strings */
         this.dyd = null;  /* dynamic structures used by the parser */
         this.source = null;  /* current source name */
@@ -187,7 +130,7 @@ const currIsNewline = function(ls) {
 };
 
 const next = function(ls) {
-    ls.current = ls.z.getc();
+    ls.current = ls.z.zgetc();
 };
 
 const save_and_next = function(ls) {
@@ -664,7 +607,6 @@ const luaX_lookahead = function(ls) {
 
 module.exports.FIRST_RESERVED   = FIRST_RESERVED;
 module.exports.LexState         = LexState;
-module.exports.MBuffer          = MBuffer;
 module.exports.RESERVED         = RESERVED;
 module.exports.isreserved       = isreserved;
 module.exports.luaX_lookahead   = luaX_lookahead;
diff --git a/src/lundump.js b/src/lundump.js
index eeb77f4..1891451 100644
--- a/src/lundump.js
+++ b/src/lundump.js
@@ -9,20 +9,20 @@ const lfunc    = require('./lfunc.js');
 const lobject  = require('./lobject.js');
 const lopcodes = require('./lopcodes.js');
 const lstring  = require('./lstring.js');
-const llex     = require('./llex.js');
+const lzio     = require('./lzio.js');
 
 let LUAC_DATA = [0x19, 0x93, defs.char["\r"], defs.char["\n"], 0x1a, defs.char["\n"]];
 
 class BytecodeParser {
 
-    constructor(L, buffer, name) {
+    constructor(L, Z, name) {
         this.intSize = 4;
         this.size_tSize = 8;
         this.instructionSize = 4;
         this.integerSize = 4;
         this.numberSize = 8;
 
-        assert(buffer instanceof llex.MBuffer, "BytecodeParser only operates on a MBuffer");
+        assert(Z instanceof lzio.ZIO, "BytecodeParser only operates on a ZIO");
         assert(Array.isArray(name));
 
         if (name[0] == defs.char["@"] || name[0] == defs.char["="])
@@ -33,32 +33,32 @@ class BytecodeParser {
             this.name = name;
 
         this.L = L;
-        this.buffer = buffer;
+        this.Z = Z;
 
         // Used to do buffer to number conversions
-        this.dv = new DataView(
-            new ArrayBuffer(
-                Math.max(this.intSize, this.size_tSize, this.instructionSize, this.integerSize, this.numberSize)
-            )
+        this.arraybuffer = new ArrayBuffer(
+            Math.max(this.intSize, this.size_tSize, this.instructionSize, this.integerSize, this.numberSize)
         );
+        this.dv = new DataView(this.arraybuffer);
+        this.u8 = new Uint8Array(this.arraybuffer);
     }
 
     read(size) {
-        let buffer = this.buffer.read(size);
-        assert(Array.isArray(buffer));
-        if (buffer.length < size) this.error("truncated");
-        return buffer;
+        let u8 = new Uint8Array(size);
+        if(lzio.luaZ_read(this.Z, u8, 0, size) !== 0)
+            this.error("truncated");
+        return Array.from(u8);
     }
 
     readByte() {
-        return this.read(1)[0];
+        if (lzio.luaZ_read(this.Z, this.u8, 0, 1) !== 0)
+            this.error("truncated");
+        return this.u8[0];
     }
 
     readInteger() {
-        let buffer = this.read(this.integerSize);
-        for (let i = 0; i < buffer.length; i++)
-            this.dv.setUint8(i, buffer[i]);
-
+        if (lzio.luaZ_read(this.Z, this.u8, 0, this.integerSize) !== 0)
+            this.error("truncated");
         return this.dv.getInt32(0, true);
     }
 
@@ -67,20 +67,14 @@ class BytecodeParser {
     }
 
     readInt() {
-        let buffer = this.read(this.intSize);
-
-        for (let i = 0; i < buffer.length; i++)
-            this.dv.setUint8(i, buffer[i]);
-
+        if (lzio.luaZ_read(this.Z, this.u8, 0, this.intSize) !== 0)
+            this.error("truncated");
         return this.dv.getInt32(0, true);
     }
 
     readNumber() {
-        let buffer = this.read(this.numberSize);
-
-        for (let i = 0; i < buffer.length; i++)
-            this.dv.setUint8(i, buffer[i]);
-
+        if (lzio.luaZ_read(this.Z, this.u8, 0, this.numberSize) !== 0)
+            this.error("truncated");
         return this.dv.getFloat64(0, true);
     }
 
@@ -94,7 +88,7 @@ class BytecodeParser {
             return null;
         }
 
-        return lstring.luaS_new(this.L, this.read(size));
+        return lstring.luaS_bless(this.L, this.read(size));
     }
 
     /* creates a mask with 'n' 1 bits at position 'p' */
@@ -108,11 +102,9 @@ class BytecodeParser {
     }
 
     readInstruction() {
-        let ins = new DataView(new ArrayBuffer(this.instructionSize));
-        for (let i = 0; i < this.instructionSize; i++)
-            ins.setUint8(i, this.readByte());
-
-        return ins.getUint32(0, true);
+        if (lzio.luaZ_read(this.Z, this.u8, 0, this.instructionSize) !== 0)
+            this.error("truncated");
+        return this.dv.getUint32(0, true);
     }
 
     readCode(f) {
@@ -269,8 +261,8 @@ class BytecodeParser {
     }
 }
 
-const luaU_undump = function(L, buffer, name) {
-    let S = new BytecodeParser(L, buffer, name);
+const luaU_undump = function(L, Z, name) {
+    let S = new BytecodeParser(L, Z, name);
     S.checkHeader();
     let cl = lfunc.luaF_newLclosure(L, S.readByte());
     L.stack[L.top++] = new lobject.TValue(defs.CT.LUA_TLCL, cl);
diff --git a/src/lzio.js b/src/lzio.js
new file mode 100644
index 0000000..919ed44
--- /dev/null
+++ b/src/lzio.js
@@ -0,0 +1,81 @@
+"use strict";
+
+const assert = require('assert');
+
+
+class MBuffer {
+    constructor() {
+        this.buffer = null;
+        this.n = 0;
+    }
+}
+
+class ZIO {
+    constructor(L, reader, data) {
+        this.L = L;           /* Lua state (for reader) */
+        assert(typeof reader == "function", "ZIO requires a reader");
+        this.reader = reader; /* reader function */
+        this.data = data;     /* additional data */
+        this.n = 0;           /* bytes still unread */
+        this.buffer = null;
+        this.off = 0;         /* current position in buffer */
+    }
+
+    zgetc () {
+        return ((this.n--) > 0) ? this.buffer[this.off++] : luaZ_fill(this);
+    }
+}
+
+const EOZ = -1;
+
+const luaZ_fill = function(z) {
+    let size;
+    let buff = z.reader(z.L, z.data);
+    if (buff === null)
+        return EOZ;
+    if (buff instanceof DataView) {
+        z.buffer = new Uint8Array(buff.buffer, buff.byteOffset, buff.byteLength);
+        z.off = 0;
+        size = buff.byteLength - buff.byteOffset;
+    } else {
+        assert(typeof buff !== "string", "Should only load binary of array of bytes");
+        z.buffer = buff;
+        z.off = 0;
+        size = buff.length;
+    }
+    if (size === 0)
+        return EOZ;
+    z.n = size - 1;
+    return z.buffer[z.off++];
+};
+
+/* b should be an array-like that will be set to bytes
+ * b_offset is the offset at which to start filling */
+const luaZ_read = function(z, b, b_offset, n) {
+    while (n) {
+        if (z.n === 0) { /* no bytes in buffer? */
+            if (luaZ_fill(z) === EOZ)
+                return n; /* no more input; return number of missing bytes */
+            else {
+                z.n++;  /* luaZ_fill consumed first byte; put it back */
+                z.off--;
+            }
+        }
+        let m = (n <= z.n) ? n : z.n; /* min. between n and z->n */
+        for (let i=0; i<m; i++) {
+            b[b_offset++] = z.buffer[z.off++];
+        }
+        z.n -= m;
+        if (z.n === 0) // remove reference to input so it can get freed
+            z.buffer = null;
+        n -= m;
+    }
+
+    return 0;
+};
+
+module.exports.EOZ       = EOZ;
+module.exports.luaZ_fill = luaZ_fill;
+module.exports.luaZ_read = luaZ_read;
+module.exports.MBuffer   = MBuffer;
+module.exports.ZIO       = ZIO;