Basic lexing tests

5 files changed, 148 insertions, 81 deletions

diff --git a/src/lfunc.js b/src/lfunc.js
index 3b3c8ce..fde910b 100644
--- a/src/lfunc.js
+++ b/src/lfunc.js
@@ -1,6 +1,8 @@
 /*jshint esversion: 6 */
 "use strict";
-const assert = require('assert');
+const assert  = require('assert');
+
+const lobject = require('./lobject.js');
 
 class Proto {
 
@@ -53,6 +55,15 @@ class UpVal {
 
 }
 
+const luaF_newLclosure = function(L, n) {
+    let c = new lobject.LClosure();
+    c.p = null;
+    c.nupvalues = n;
+    while (n--) c.upvals[n] = null;
+    return c;
+};
+
+
 const findupval = function(L, level) {
     let pp = L.openupval;
     
@@ -116,10 +127,11 @@ const luaF_getlocalname = function(f, local_number, pc) {
 }
 
 
+module.exports.MAXUPVAL          = 255;
 module.exports.Proto             = Proto;
 module.exports.UpVal             = UpVal;
 module.exports.findupval         = findupval;
 module.exports.luaF_close        = luaF_close;
-module.exports.MAXUPVAL          = 255;
+module.exports.luaF_getlocalname = luaF_getlocalname
 module.exports.luaF_initupvals   = luaF_initupvals;
-module.exports.luaF_getlocalname = luaF_getlocalname
\ No newline at end of file
+module.exports.luaF_newLclosure  = luaF_newLclosure;
\ No newline at end of file
diff --git a/src/ljstype.js b/src/ljstype.js
index 52ac153..192db6f 100644
--- a/src/ljstype.js
+++ b/src/ljstype.js
@@ -5,6 +5,7 @@ const assert = require('assert');
 
 const lisdigit = function(c) {
     return /^\d$/.test(c.charAt(0));
+};
 
 const lisxdigit = function(c) {
     return /^[0-9a-fA-F]$/.test(c.charAt(0));
@@ -18,12 +19,12 @@ const lislalpha = function(c) {
     return /^[_a-zA-z]$/.test(c.charAt(0));
 };
 
-const lislanum = function(c) {
+const lislalnum = function(c) {
     return /^[_a-zA-z0-9]$/.test(c.charAt(0));
 };
 
-module.exports.lisdigit  = lisdigit;
-module.exports.lislalpha = lislalpha;
-module.exports.lislanum  = lislanum;
-module.exports.lisspace  = lisspace;
-module.exports.lisxdigit = lisxdigit;
\ No newline at end of file
+module.exports.lisdigit   = lisdigit;
+module.exports.lislalnum  = lislalnum;
+module.exports.lislalpha  = lislalpha;
+module.exports.lisspace   = lisspace;
+module.exports.lisxdigit  = lisxdigit;
\ No newline at end of file
diff --git a/src/llex.js b/src/llex.js
index 4c85f65..1adac0a 100644
--- a/src/llex.js
+++ b/src/llex.js
@@ -4,11 +4,12 @@
 const assert  = require('assert');
 
 const lapi    = require('./lapi.js');
+const lauxlib = require('./lauxlib.js');
 const ldebug  = require('./ldebug.js');
 const ldo     = require('./ldo.js');
-const lua     = require('./lua.js');
-const lobject = require('./lobject');
 const ljstype = require('./ljstype');
+const lobject = require('./lobject');
+const lua     = require('./lua.js');
 const TValue  = lobject.TValue;
 const CT      = lua.constant_types;
 const TS      = lua.thread_status;
@@ -24,73 +25,41 @@ const RESERVED = {
     TK_ELSEIF:   FIRST_RESERVED + 4,
     TK_END:      FIRST_RESERVED + 5,
     TK_FALSE:    FIRST_RESERVED + 6,
-    TK_FOR:      FIRST_RESERVED + 8,
-    TK_FUNCTION: FIRST_RESERVED + 10,
-    TK_GOTO:     FIRST_RESERVED + 11,
-    TK_IF:       FIRST_RESERVED + 12,
-    TK_IN:       FIRST_RESERVED + 13,
-    TK_LOCAL:    FIRST_RESERVED + 14,
-    TK_NIL:      FIRST_RESERVED + 15,
-    TK_NOT:      FIRST_RESERVED + 16,
-    TK_OR:       FIRST_RESERVED + 17,
-    TK_REPEAT:   FIRST_RESERVED + 18,
-    TK_RETURN:   FIRST_RESERVED + 19,
-    TK_THEN:     FIRST_RESERVED + 20,
-    TK_TRUE:     FIRST_RESERVED + 21,
-    TK_UNTIL:    FIRST_RESERVED + 22,
-    TK_WHILE:    FIRST_RESERVED + 23,
+    TK_FOR:      FIRST_RESERVED + 7,
+    TK_FUNCTION: FIRST_RESERVED + 8,
+    TK_GOTO:     FIRST_RESERVED + 9,
+    TK_IF:       FIRST_RESERVED + 10,
+    TK_IN:       FIRST_RESERVED + 11,
+    TK_LOCAL:    FIRST_RESERVED + 12,
+    TK_NIL:      FIRST_RESERVED + 13,
+    TK_NOT:      FIRST_RESERVED + 14,
+    TK_OR:       FIRST_RESERVED + 15,
+    TK_REPEAT:   FIRST_RESERVED + 16,
+    TK_RETURN:   FIRST_RESERVED + 17,
+    TK_THEN:     FIRST_RESERVED + 18,
+    TK_TRUE:     FIRST_RESERVED + 19,
+    TK_UNTIL:    FIRST_RESERVED + 20,
+    TK_WHILE:    FIRST_RESERVED + 21,
     /* other terminal symbols */
-    TK_IDIV:     FIRST_RESERVED + 24,
-    TK_CONCAT:   FIRST_RESERVED + 25,
-    TK_DOTS:     FIRST_RESERVED + 26,
-    TK_EQ:       FIRST_RESERVED + 27,
-    TK_GE:       FIRST_RESERVED + 28,
-    TK_LE:       FIRST_RESERVED + 29,
-    TK_NE:       FIRST_RESERVED + 30,
-    TK_SHL:      FIRST_RESERVED + 31,
-    TK_SHR:      FIRST_RESERVED + 32,
-    TK_DBCOLON:  FIRST_RESERVED + 33,
-    TK_EOS:      FIRST_RESERVED + 34,
-    TK_FLT:      FIRST_RESERVED + 35,
-    TK_INT:      FIRST_RESERVED + 36,
-    TK_NAME:     FIRST_RESERVED + 37,
-    TK_STRING:   FIRST_RESERVED + 38
+    TK_IDIV:     FIRST_RESERVED + 22,
+    TK_CONCAT:   FIRST_RESERVED + 23,
+    TK_DOTS:     FIRST_RESERVED + 24,
+    TK_EQ:       FIRST_RESERVED + 25,
+    TK_GE:       FIRST_RESERVED + 26,
+    TK_LE:       FIRST_RESERVED + 27,
+    TK_NE:       FIRST_RESERVED + 28,
+    TK_SHL:      FIRST_RESERVED + 29,
+    TK_SHR:      FIRST_RESERVED + 30,
+    TK_DBCOLON:  FIRST_RESERVED + 31,
+    TK_EOS:      FIRST_RESERVED + 32,
+    TK_FLT:      FIRST_RESERVED + 33,
+    TK_INT:      FIRST_RESERVED + 34,
+    TK_NAME:     FIRST_RESERVED + 35,
+    TK_STRING:   FIRST_RESERVED + 36
 };
 
 const R = RESERVED;
 
-const reserved_keywords = [
-    "and", "break", "do", "else", "elseif",
-    "end", "false", "for", "function", "goto", "if",
-    "in", "local", "nil", "not", "or", "repeat",
-    "return", "then", "true", "until", "while"
-];
-
-const reserved_keywords_tokens = [
-    R.TK_AND,
-    R.TK_BREAK,
-    R.TK_DO,
-    R.TK_ELSE,
-    R.TK_ELSEIF,
-    R.TK_END,
-    R.TK_FALSE,
-    R.TK_FOR,
-    R.TK_FUNCTION,
-    R.TK_GOTO,
-    R.TK_IF,
-    R.TK_IN,
-    R.TK_LOCAL,
-    R.TK_NIL,
-    R.TK_NOT,
-    R.TK_OR,
-    R.TK_REPEAT,
-    R.TK_RETURN,
-    R.TK_THEN,
-    R.TK_TRUE,
-    R.TK_UNTIL,
-    R.TK_WHILE,
-];
-
 const luaX_tokens = [
     "and", "break", "do", "else", "elseif",
     "end", "false", "for", "function", "goto", "if",
@@ -103,7 +72,7 @@ const luaX_tokens = [
 
 const NUM_RESERVED = Object.keys(RESERVED).length;
 
-class Buffer {
+class MBuffer {
     constructor(string) {
         this.buffer = string ? string.split('') : [];
         this.n = this.buffer.length;
@@ -137,8 +106,8 @@ class LexState {
         this.lookahead = null;  /* look ahead token */
         this.fs = null;  /* current function (parser) */
         this.L = null;
-        this.z = new Buffer();
-        this.buff = new Buffer();  /* buffer for tokens */
+        this.z = new MBuffer();
+        this.buff = new MBuffer();  /* buffer for tokens */
         this.h = null;  /* to avoid collection/reuse strings */
         this.dyd = null;  /* dynamic structures used by the parser */
         this.source = null;  /* current source name */
@@ -195,10 +164,24 @@ const inclinenumber = function(ls) {
 };
 
 const luaX_setinput = function(L, ls, z, source, firstchar) {
-    ls.t.token = 0;
+    ls.t = {
+        token: 0,
+        seminfo: {
+            i: NaN,
+            r: NaN,
+            ts: null
+        }
+    };
     ls.L = L;
     ls.current = firstchar;
-    ls.lookahead.token = R.TK_EOS;
+    ls.lookahead = {
+        token: R.TK_EOS,
+        seminfo: {
+            i: NaN,
+            r: NaN,
+            ts: null
+        }
+    };
     ls.z = z;
     ls.fs = null;
     ls.linenumber = 1;
@@ -562,8 +545,9 @@ const llex = function(ls, seminfo) {
 
                     let ts = new TValue(CT.LUA_TLNGSTR, ls.buff.buffer.join(''));
                     seminfo.ts = ts;
-                    if (reserved_keywords.indexOf(ts.value) >= 0)  /* reserved word? */
-                        return reserved_keywords_tokens[reserved_keywords.indexOf(ts.value)];
+                    let kidx = luaX_tokens.slice(0, 22).indexOf(ts.value)
+                    if (kidx >= 0)  /* reserved word? */
+                        return kidx + FIRST_RESERVED;
                     else
                         return R.TK_NAME;
                 } else {  /* single-char tokens (+ - / ...) */
@@ -591,6 +575,11 @@ const luaX_lookahead = function(ls) {
     return ls.lookahead.token;
 };
 
+module.exports.FIRST_RESERVED = FIRST_RESERVED;
+module.exports.LexState       = LexState;
 module.exports.luaX_lookahead = luaX_lookahead;
 module.exports.luaX_next      = luaX_next;
-module.exports.luaX_setinput  = luaX_setinput;
\ No newline at end of file
+module.exports.luaX_setinput  = luaX_setinput;
+module.exports.MBuffer        = MBuffer;
+module.exports.RESERVED       = RESERVED;
+module.exports.luaX_tokens    = luaX_tokens;
\ No newline at end of file
diff --git a/src/lparser.js b/src/lparser.js
new file mode 100644
index 0000000..a82548c
--- /dev/null
+++ b/src/lparser.js
@@ -0,0 +1,15 @@
+/* jshint esversion: 6 */
+"use strict";
+
+const assert = require('assert');
+
+const llex   = require('./llex.js');
+const lfunc  = require('./lfunc.js');
+
+
+const luaY_parser = function(L, z, buff, dyd, name, firstchar) {
+    // TODO ...
+};
+
+
+module.exports.luaY_parser = luaY_parser;
\ No newline at end of file
diff --git a/tests/llex.js b/tests/llex.js
new file mode 100644
index 0000000..3c05366
--- /dev/null
+++ b/tests/llex.js
@@ -0,0 +1,50 @@
+/*jshint esversion: 6 */
+"use strict";
+
+const test     = require('tape');
+const beautify = require('js-beautify').js_beautify;
+
+const tests    = require("./tests.js");
+
+const lapi     = require("../src/lapi.js");
+const lauxlib  = require("../src/lauxlib.js");
+const llex     = require("../src/llex.js");
+const lua      = require('../src/lua.js');
+const R        = llex.RESERVED;
+
+
+test('basic lexing', function (t) {
+    let luaCode = `
+        return "hello lex !"
+    `, L;
+    
+    t.plan(2);
+
+    let readTokens = [];
+
+    t.doesNotThrow(function () {
+
+        L = lauxlib.luaL_newstate();
+
+        let ls = new llex.LexState();
+        llex.luaX_setinput(L, ls, new llex.MBuffer(luaCode), luaCode, luaCode.charAt(0));
+
+        llex.luaX_next(ls);
+
+        while (ls.t.token !== R.TK_EOS) {
+            console.log(llex.luaX_tokens[ls.t.token - llex.FIRST_RESERVED]);
+
+            readTokens.push(ls.t.token);
+            llex.luaX_next(ls);
+        }
+
+
+    }, "JS Lua program ran without error");
+
+    t.deepEqual(
+        readTokens,
+        [R.TK_RETURN, R.TK_STRING],
+        "Correct tokens found"
+    )
+
+});
\ No newline at end of file