From 5a47025d64f013975051473c1115ff70c0281785 Mon Sep 17 00:00:00 2001 From: Benoit Giannangeli Date: Mon, 27 Feb 2017 08:46:56 +0100 Subject: lexing --- src/ljstype.js | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/ljstype.js (limited to 'src/ljstype.js') diff --git a/src/ljstype.js b/src/ljstype.js new file mode 100644 index 0000000..52ac153 --- /dev/null +++ b/src/ljstype.js @@ -0,0 +1,29 @@ +/* jshint esversion: 6 */ +"use strict"; + +const assert = require('assert'); + +const lisdigit = function(c) { + return /^\d$/.test(c.charAt(0)); + +const lisxdigit = function(c) { + return /^[0-9a-fA-F]$/.test(c.charAt(0)); +}; + +const lisspace = function(c) { + return /^\s$/.test(c.charAt(0)); +}; + +const lislalpha = function(c) { + return /^[_a-zA-z]$/.test(c.charAt(0)); +}; + +const lislanum = function(c) { + return /^[_a-zA-z0-9]$/.test(c.charAt(0)); +}; + +module.exports.lisdigit = lisdigit; +module.exports.lislalpha = lislalpha; +module.exports.lislanum = lislanum; +module.exports.lisspace = lisspace; +module.exports.lisxdigit = lisxdigit; \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 6beeccbd0a859f3a9d1be4142d16a3d11ac30743 Mon Sep 17 00:00:00 2001 From: Benoit Giannangeli Date: Mon, 27 Feb 2017 15:08:09 +0100 Subject: Basic lexing tests --- src/lfunc.js | 18 ++++++-- src/ljstype.js | 13 +++--- src/llex.js | 133 ++++++++++++++++++++++++++------------------------------- src/lparser.js | 15 +++++++ tests/llex.js | 50 ++++++++++++++++++++++ 5 files changed, 148 insertions(+), 81 deletions(-) create mode 100644 src/lparser.js create mode 100644 tests/llex.js (limited to 'src/ljstype.js') diff --git a/src/lfunc.js b/src/lfunc.js index 3b3c8ce..fde910b 100644 --- a/src/lfunc.js +++ b/src/lfunc.js @@ -1,6 +1,8 @@ /*jshint esversion: 6 */ "use strict"; -const assert = require('assert'); +const assert = require('assert'); + +const lobject = require('./lobject.js'); class Proto { @@ -53,6 +55,15 @@ class UpVal { } +const luaF_newLclosure = function(L, n) { + let c = new lobject.LClosure(); + c.p = null; + c.nupvalues = n; + while (n--) c.upvals[n] = null; + return c; +}; + + const findupval = function(L, level) { let pp = L.openupval; @@ -116,10 +127,11 @@ const luaF_getlocalname = function(f, local_number, pc) { } +module.exports.MAXUPVAL = 255; module.exports.Proto = Proto; module.exports.UpVal = UpVal; module.exports.findupval = findupval; module.exports.luaF_close = luaF_close; -module.exports.MAXUPVAL = 255; +module.exports.luaF_getlocalname = luaF_getlocalname module.exports.luaF_initupvals = luaF_initupvals; -module.exports.luaF_getlocalname = luaF_getlocalname \ No newline at end of file +module.exports.luaF_newLclosure = luaF_newLclosure; \ No newline at end of file diff --git a/src/ljstype.js b/src/ljstype.js index 52ac153..192db6f 100644 --- a/src/ljstype.js +++ b/src/ljstype.js @@ -5,6 +5,7 @@ const assert = require('assert'); const lisdigit = function(c) { return /^\d$/.test(c.charAt(0)); +}; const lisxdigit = function(c) { return /^[0-9a-fA-F]$/.test(c.charAt(0)); @@ -18,12 +19,12 @@ const lislalpha = function(c) { return /^[_a-zA-z]$/.test(c.charAt(0)); }; -const lislanum = function(c) { +const lislalnum = function(c) { return /^[_a-zA-z0-9]$/.test(c.charAt(0)); }; -module.exports.lisdigit = lisdigit; -module.exports.lislalpha = lislalpha; -module.exports.lislanum = lislanum; -module.exports.lisspace = lisspace; -module.exports.lisxdigit = lisxdigit; \ No newline at end of file +module.exports.lisdigit = lisdigit; +module.exports.lislalnum = lislalnum; +module.exports.lislalpha = lislalpha; +module.exports.lisspace = lisspace; +module.exports.lisxdigit = lisxdigit; \ No newline at end of file diff --git a/src/llex.js b/src/llex.js index 4c85f65..1adac0a 100644 --- a/src/llex.js +++ b/src/llex.js @@ -4,11 +4,12 @@ const assert = require('assert'); const lapi = require('./lapi.js'); +const lauxlib = require('./lauxlib.js'); const ldebug = require('./ldebug.js'); const ldo = require('./ldo.js'); -const lua = require('./lua.js'); -const lobject = require('./lobject'); const ljstype = require('./ljstype'); +const lobject = require('./lobject'); +const lua = require('./lua.js'); const TValue = lobject.TValue; const CT = lua.constant_types; const TS = lua.thread_status; @@ -24,73 +25,41 @@ const RESERVED = { TK_ELSEIF: FIRST_RESERVED + 4, TK_END: FIRST_RESERVED + 5, TK_FALSE: FIRST_RESERVED + 6, - TK_FOR: FIRST_RESERVED + 8, - TK_FUNCTION: FIRST_RESERVED + 10, - TK_GOTO: FIRST_RESERVED + 11, - TK_IF: FIRST_RESERVED + 12, - TK_IN: FIRST_RESERVED + 13, - TK_LOCAL: FIRST_RESERVED + 14, - TK_NIL: FIRST_RESERVED + 15, - TK_NOT: FIRST_RESERVED + 16, - TK_OR: FIRST_RESERVED + 17, - TK_REPEAT: FIRST_RESERVED + 18, - TK_RETURN: FIRST_RESERVED + 19, - TK_THEN: FIRST_RESERVED + 20, - TK_TRUE: FIRST_RESERVED + 21, - TK_UNTIL: FIRST_RESERVED + 22, - TK_WHILE: FIRST_RESERVED + 23, + TK_FOR: FIRST_RESERVED + 7, + TK_FUNCTION: FIRST_RESERVED + 8, + TK_GOTO: FIRST_RESERVED + 9, + TK_IF: FIRST_RESERVED + 10, + TK_IN: FIRST_RESERVED + 11, + TK_LOCAL: FIRST_RESERVED + 12, + TK_NIL: FIRST_RESERVED + 13, + TK_NOT: FIRST_RESERVED + 14, + TK_OR: FIRST_RESERVED + 15, + TK_REPEAT: FIRST_RESERVED + 16, + TK_RETURN: FIRST_RESERVED + 17, + TK_THEN: FIRST_RESERVED + 18, + TK_TRUE: FIRST_RESERVED + 19, + TK_UNTIL: FIRST_RESERVED + 20, + TK_WHILE: FIRST_RESERVED + 21, /* other terminal symbols */ - TK_IDIV: FIRST_RESERVED + 24, - TK_CONCAT: FIRST_RESERVED + 25, - TK_DOTS: FIRST_RESERVED + 26, - TK_EQ: FIRST_RESERVED + 27, - TK_GE: FIRST_RESERVED + 28, - TK_LE: FIRST_RESERVED + 29, - TK_NE: FIRST_RESERVED + 30, - TK_SHL: FIRST_RESERVED + 31, - TK_SHR: FIRST_RESERVED + 32, - TK_DBCOLON: FIRST_RESERVED + 33, - TK_EOS: FIRST_RESERVED + 34, - TK_FLT: FIRST_RESERVED + 35, - TK_INT: FIRST_RESERVED + 36, - TK_NAME: FIRST_RESERVED + 37, - TK_STRING: FIRST_RESERVED + 38 + TK_IDIV: FIRST_RESERVED + 22, + TK_CONCAT: FIRST_RESERVED + 23, + TK_DOTS: FIRST_RESERVED + 24, + TK_EQ: FIRST_RESERVED + 25, + TK_GE: FIRST_RESERVED + 26, + TK_LE: FIRST_RESERVED + 27, + TK_NE: FIRST_RESERVED + 28, + TK_SHL: FIRST_RESERVED + 29, + TK_SHR: FIRST_RESERVED + 30, + TK_DBCOLON: FIRST_RESERVED + 31, + TK_EOS: FIRST_RESERVED + 32, + TK_FLT: FIRST_RESERVED + 33, + TK_INT: FIRST_RESERVED + 34, + TK_NAME: FIRST_RESERVED + 35, + TK_STRING: FIRST_RESERVED + 36 }; const R = RESERVED; -const reserved_keywords = [ - "and", "break", "do", "else", "elseif", - "end", "false", "for", "function", "goto", "if", - "in", "local", "nil", "not", "or", "repeat", - "return", "then", "true", "until", "while" -]; - -const reserved_keywords_tokens = [ - R.TK_AND, - R.TK_BREAK, - R.TK_DO, - R.TK_ELSE, - R.TK_ELSEIF, - R.TK_END, - R.TK_FALSE, - R.TK_FOR, - R.TK_FUNCTION, - R.TK_GOTO, - R.TK_IF, - R.TK_IN, - R.TK_LOCAL, - R.TK_NIL, - R.TK_NOT, - R.TK_OR, - R.TK_REPEAT, - R.TK_RETURN, - R.TK_THEN, - R.TK_TRUE, - R.TK_UNTIL, - R.TK_WHILE, -]; - const luaX_tokens = [ "and", "break", "do", "else", "elseif", "end", "false", "for", "function", "goto", "if", @@ -103,7 +72,7 @@ const luaX_tokens = [ const NUM_RESERVED = Object.keys(RESERVED).length; -class Buffer { +class MBuffer { constructor(string) { this.buffer = string ? string.split('') : []; this.n = this.buffer.length; @@ -137,8 +106,8 @@ class LexState { this.lookahead = null; /* look ahead token */ this.fs = null; /* current function (parser) */ this.L = null; - this.z = new Buffer(); - this.buff = new Buffer(); /* buffer for tokens */ + this.z = new MBuffer(); + this.buff = new MBuffer(); /* buffer for tokens */ this.h = null; /* to avoid collection/reuse strings */ this.dyd = null; /* dynamic structures used by the parser */ this.source = null; /* current source name */ @@ -195,10 +164,24 @@ const inclinenumber = function(ls) { }; const luaX_setinput = function(L, ls, z, source, firstchar) { - ls.t.token = 0; + ls.t = { + token: 0, + seminfo: { + i: NaN, + r: NaN, + ts: null + } + }; ls.L = L; ls.current = firstchar; - ls.lookahead.token = R.TK_EOS; + ls.lookahead = { + token: R.TK_EOS, + seminfo: { + i: NaN, + r: NaN, + ts: null + } + }; ls.z = z; ls.fs = null; ls.linenumber = 1; @@ -562,8 +545,9 @@ const llex = function(ls, seminfo) { let ts = new TValue(CT.LUA_TLNGSTR, ls.buff.buffer.join('')); seminfo.ts = ts; - if (reserved_keywords.indexOf(ts.value) >= 0) /* reserved word? */ - return reserved_keywords_tokens[reserved_keywords.indexOf(ts.value)]; + let kidx = luaX_tokens.slice(0, 22).indexOf(ts.value) + if (kidx >= 0) /* reserved word? */ + return kidx + FIRST_RESERVED; else return R.TK_NAME; } else { /* single-char tokens (+ - / ...) */ @@ -591,6 +575,11 @@ const luaX_lookahead = function(ls) { return ls.lookahead.token; }; +module.exports.FIRST_RESERVED = FIRST_RESERVED; +module.exports.LexState = LexState; module.exports.luaX_lookahead = luaX_lookahead; module.exports.luaX_next = luaX_next; -module.exports.luaX_setinput = luaX_setinput; \ No newline at end of file +module.exports.luaX_setinput = luaX_setinput; +module.exports.MBuffer = MBuffer; +module.exports.RESERVED = RESERVED; +module.exports.luaX_tokens = luaX_tokens; \ No newline at end of file diff --git a/src/lparser.js b/src/lparser.js new file mode 100644 index 0000000..a82548c --- /dev/null +++ b/src/lparser.js @@ -0,0 +1,15 @@ +/* jshint esversion: 6 */ +"use strict"; + +const assert = require('assert'); + +const llex = require('./llex.js'); +const lfunc = require('./lfunc.js'); + + +const luaY_parser = function(L, z, buff, dyd, name, firstchar) { + // TODO ... +}; + + +module.exports.luaY_parser = luaY_parser; \ No newline at end of file diff --git a/tests/llex.js b/tests/llex.js new file mode 100644 index 0000000..3c05366 --- /dev/null +++ b/tests/llex.js @@ -0,0 +1,50 @@ +/*jshint esversion: 6 */ +"use strict"; + +const test = require('tape'); +const beautify = require('js-beautify').js_beautify; + +const tests = require("./tests.js"); + +const lapi = require("../src/lapi.js"); +const lauxlib = require("../src/lauxlib.js"); +const llex = require("../src/llex.js"); +const lua = require('../src/lua.js'); +const R = llex.RESERVED; + + +test('basic lexing', function (t) { + let luaCode = ` + return "hello lex !" + `, L; + + t.plan(2); + + let readTokens = []; + + t.doesNotThrow(function () { + + L = lauxlib.luaL_newstate(); + + let ls = new llex.LexState(); + llex.luaX_setinput(L, ls, new llex.MBuffer(luaCode), luaCode, luaCode.charAt(0)); + + llex.luaX_next(ls); + + while (ls.t.token !== R.TK_EOS) { + console.log(llex.luaX_tokens[ls.t.token - llex.FIRST_RESERVED]); + + readTokens.push(ls.t.token); + llex.luaX_next(ls); + } + + + }, "JS Lua program ran without error"); + + t.deepEqual( + readTokens, + [R.TK_RETURN, R.TK_STRING], + "Correct tokens found" + ) + +}); \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 8d576081a53ee1a866491b423c5373951aa6a7c4 Mon Sep 17 00:00:00 2001 From: Benoit Giannangeli Date: Tue, 28 Feb 2017 11:29:37 +0100 Subject: jshint --- .jshintrc | 6 ++++++ src/lapi.js | 1 - src/lbaselib.js | 1 - src/lcode.js | 9 ++++----- src/lcorolib.js | 1 - src/linit.js | 1 - src/ljstype.js | 1 - src/llex.js | 3 +-- src/lmathlib.js | 1 - src/lparser.js | 42 +++++++++++++++++++++++++++++------------- src/ltablib.js | 1 - 11 files changed, 40 insertions(+), 27 deletions(-) create mode 100644 .jshintrc (limited to 'src/ljstype.js') diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 0000000..e64eb7e --- /dev/null +++ b/.jshintrc @@ -0,0 +1,6 @@ +{ + "node": true, + "esversion": 6, + "indent": false, + "white": false +} \ No newline at end of file diff --git a/src/lapi.js b/src/lapi.js index d0c099f..c4a7bbe 100644 --- a/src/lapi.js +++ b/src/lapi.js @@ -1,4 +1,3 @@ -/* jshint esversion: 6 */ "use strict"; const assert = require('assert'); diff --git a/src/lbaselib.js b/src/lbaselib.js index 012f5c3..b663b87 100644 --- a/src/lbaselib.js +++ b/src/lbaselib.js @@ -1,4 +1,3 @@ -/* jshint esversion: 6 */ "use strict"; const assert = require('assert'); diff --git a/src/lcode.js b/src/lcode.js index e64c0a9..9a5cda0 100644 --- a/src/lcode.js +++ b/src/lcode.js @@ -1,4 +1,3 @@ -/* jshint esversion: 6 */ "use strict"; const assert = require('assert'); @@ -203,7 +202,7 @@ const luaK_patchclose = function(fs, list, level) { level++; /* argument is +1 to reserve 0 as non-op */ for (; list !== NO_JUMP; list = getjump(fs, list)) { let ins = fs.f.code[list]; - assert(ins.opcode === OpCodesI.OP_JMP && (ins.A === 0 || i.A >= level); + assert(ins.opcode === OpCodesI.OP_JMP && (ins.A === 0 || ins.A >= level)); lopcode.SETARG_A(ins, level); } }; @@ -237,9 +236,9 @@ const luaK_codeABC = function(fs, o, a, b, c) { ** Format and emit an 'iABx' instruction. */ const luaK_codeABx = function(fs, o, a, bc) { - lua_assert(lopcode.getOpMode(o) == lopcode.iABx || getOpMode(o) == lopcode.iAsBx); - lua_assert(lopcode.getCMode(o) == lopcode.OpArgN); - lua_assert(a <= lopcode.MAXARG_A && bc <= lopcode.MAXARG_Bx); + assert(lopcode.getOpMode(o) == lopcode.iABx || lopcode.getOpMode(o) == lopcode.iAsBx); + assert(lopcode.getCMode(o) == lopcode.OpArgN); + assert(a <= lopcode.MAXARG_A && bc <= lopcode.MAXARG_Bx); return luaK_code(fs, lopcode.CREATE_ABx(o, a, bc)); }; diff --git a/src/lcorolib.js b/src/lcorolib.js index 05f3b00..b2d0de8 100644 --- a/src/lcorolib.js +++ b/src/lcorolib.js @@ -1,4 +1,3 @@ -/* jshint esversion: 6 */ "use strict"; const assert = require('assert'); diff --git a/src/linit.js b/src/linit.js index 36c0121..b926683 100644 --- a/src/linit.js +++ b/src/linit.js @@ -1,4 +1,3 @@ -/* jshint esversion: 6 */ "use strict"; const assert = require('assert'); diff --git a/src/ljstype.js b/src/ljstype.js index 192db6f..a92c8d0 100644 --- a/src/ljstype.js +++ b/src/ljstype.js @@ -1,4 +1,3 @@ -/* jshint esversion: 6 */ "use strict"; const assert = require('assert'); diff --git a/src/llex.js b/src/llex.js index 1992856..59bc4a4 100644 --- a/src/llex.js +++ b/src/llex.js @@ -1,4 +1,3 @@ -/* jshint esversion: 6 */ "use strict"; const assert = require('assert'); @@ -265,7 +264,7 @@ const lexerror = function(ls, msg, token) { const luaX_syntaxerror = function(ls, msg) { lexerror(ls, msg, ls.t.token); -} +}; /* ** skip a sequence '[=*[' or ']=*]'; if sequence is well formed, return diff --git a/src/lmathlib.js b/src/lmathlib.js index 54bdc46..55f8416 100644 --- a/src/lmathlib.js +++ b/src/lmathlib.js @@ -1,4 +1,3 @@ -/* jshint esversion: 6 */ "use strict"; const assert = require('assert'); diff --git a/src/lparser.js b/src/lparser.js index 807e8e7..a7e4209 100644 --- a/src/lparser.js +++ b/src/lparser.js @@ -1,4 +1,3 @@ -/* jshint esversion: 6 */ "use strict"; const assert = require('assert'); @@ -20,6 +19,12 @@ const Table = lobject.Table; const UnOpr = lcode.UnOpr; const UpVal = lfunc.UpVal; +const MAXVARS = 200; + +const hasmultret = function(k) { + return k == expkind.VCALL || k == expkind.VVARARG; +}; + class BlockCnt { constructor() { this.previous = null; /* chain */ @@ -376,6 +381,16 @@ const close_func = function(ls) { /* GRAMMAR RULES */ /*============================================================*/ +const block_follow = function(ls, withuntil) { + switch (ls.t.token) { + case R.TK_ELSE: case R.TK_ELSEIF: + case R.TK_END: case R.TK_EOS: + return true; + case R.TK_UNTIL: return withuntil; + default: return false; + } +}; + const statlist = function(ls) { /* statlist -> { stat [';'] } */ while (!block_follow(ls, 1)) { @@ -436,7 +451,7 @@ const closelistfield = function(fs, cc) { lcode.luaK_exp2nextreg(fs, cc.v); cc.v.k = expkind.VVOID; if (cc.tostore === lopcode.LFIELDS_PER_FLUSH) { - luaK_setlist(fs, cc.t.u.info, cc.na, cc.tostore); /* flush */ + lcode.luaK_setlist(fs, cc.t.u.info, cc.na, cc.tostore); /* flush */ cc.tostore = 0; /* no more items pending */ } }; @@ -548,6 +563,7 @@ const simpleexp = function(ls, v) { let fs = ls.fs; check_condition(ls, fs.f.is_vararg, "cannot use '...' outside a vararg function"); init_exp(v, expkind.VVARARG, lcode.luaK_codeABC(fs, OpCodesI.OP_VARARG, 0, 1, 0)); + break; } case '{': { /* constructor */ constructor(ls, v); @@ -604,16 +620,16 @@ const getbinopr = function(op) { }; const priority = [ /* ORDER OPR */ - {10, 10}, {10, 10}, /* '+' '-' */ - {11, 11}, {11, 11}, /* '*' '%' */ - {14, 13}, /* '^' (right associative) */ - {11, 11}, {11, 11}, /* '/' '//' */ - {6, 6}, {4, 4}, {5, 5}, /* '&' '|' '~' */ - {7, 7}, {7, 7}, /* '<<' '>>' */ - {9, 8}, /* '..' (right associative) */ - {3, 3}, {3, 3}, {3, 3}, /* ==, <, <= */ - {3, 3}, {3, 3}, {3, 3}, /* ~=, >, >= */ - {2, 2}, {1, 1} /* and, or */ + {left: 10, right: 10}, {left: 10, right: 10}, /* '+' '-' */ + {left: 11, right: 11}, {left: 11, right: 11}, /* '*' '%' */ + {left: 14, right: 13}, /* '^' (right associative) */ + {left: 11, right: 11}, {left: 11, right: 11}, /* '/' '//' */ + {left: 6, right: 6}, {left: 4, right: 4}, {left: 5, right: 5}, /* '&' '|' '~' */ + {left: 7, right: 7}, {left: 7, right: 7}, /* '<<' '>>' */ + {left: 9, right: 8}, /* '..' (right associative) */ + {left: 3, right: 3}, {left: 3, right: 3}, {left: 3, right: 3}, /* ==, <, <= */ + {left: 3, right: 3}, {left: 3, right: 3}, {left: 3, right: 3}, /* ~=, >, >= */ + {left: 2, right: 2}, {left: 1, right: 1} /* and, or */ ]; const UNARY_PRIORITY = 12; @@ -650,7 +666,7 @@ const subexpr = function(ls, v, limit) { const expr = function(ls, v) { subexpr(ls, v, 0); -} +}; const test_then_block = function(ls, escapelist) { /* test_then_block -> [IF | ELSEIF] cond THEN block */ diff --git a/src/ltablib.js b/src/ltablib.js index fa43523..a7963d3 100644 --- a/src/ltablib.js +++ b/src/ltablib.js @@ -1,4 +1,3 @@ -/* jshint esversion: 6 */ "use strict"; const assert = require('assert'); -- cgit v1.2.3-54-g00ecf