From 6beeccbd0a859f3a9d1be4142d16a3d11ac30743 Mon Sep 17 00:00:00 2001 From: Benoit Giannangeli Date: Mon, 27 Feb 2017 15:08:09 +0100 Subject: Basic lexing tests --- src/lfunc.js | 18 ++++++-- src/ljstype.js | 13 +++--- src/llex.js | 133 ++++++++++++++++++++++++++------------------------------- src/lparser.js | 15 +++++++ tests/llex.js | 50 ++++++++++++++++++++++ 5 files changed, 148 insertions(+), 81 deletions(-) create mode 100644 src/lparser.js create mode 100644 tests/llex.js diff --git a/src/lfunc.js b/src/lfunc.js index 3b3c8ce..fde910b 100644 --- a/src/lfunc.js +++ b/src/lfunc.js @@ -1,6 +1,8 @@ /*jshint esversion: 6 */ "use strict"; -const assert = require('assert'); +const assert = require('assert'); + +const lobject = require('./lobject.js'); class Proto { @@ -53,6 +55,15 @@ class UpVal { } +const luaF_newLclosure = function(L, n) { + let c = new lobject.LClosure(); + c.p = null; + c.nupvalues = n; + while (n--) c.upvals[n] = null; + return c; +}; + + const findupval = function(L, level) { let pp = L.openupval; @@ -116,10 +127,11 @@ const luaF_getlocalname = function(f, local_number, pc) { } +module.exports.MAXUPVAL = 255; module.exports.Proto = Proto; module.exports.UpVal = UpVal; module.exports.findupval = findupval; module.exports.luaF_close = luaF_close; -module.exports.MAXUPVAL = 255; +module.exports.luaF_getlocalname = luaF_getlocalname module.exports.luaF_initupvals = luaF_initupvals; -module.exports.luaF_getlocalname = luaF_getlocalname \ No newline at end of file +module.exports.luaF_newLclosure = luaF_newLclosure; \ No newline at end of file diff --git a/src/ljstype.js b/src/ljstype.js index 52ac153..192db6f 100644 --- a/src/ljstype.js +++ b/src/ljstype.js @@ -5,6 +5,7 @@ const assert = require('assert'); const lisdigit = function(c) { return /^\d$/.test(c.charAt(0)); +}; const lisxdigit = function(c) { return /^[0-9a-fA-F]$/.test(c.charAt(0)); @@ -18,12 +19,12 @@ const lislalpha = function(c) { return /^[_a-zA-z]$/.test(c.charAt(0)); }; -const lislanum = function(c) { +const lislalnum = function(c) { return /^[_a-zA-z0-9]$/.test(c.charAt(0)); }; -module.exports.lisdigit = lisdigit; -module.exports.lislalpha = lislalpha; -module.exports.lislanum = lislanum; -module.exports.lisspace = lisspace; -module.exports.lisxdigit = lisxdigit; \ No newline at end of file +module.exports.lisdigit = lisdigit; +module.exports.lislalnum = lislalnum; +module.exports.lislalpha = lislalpha; +module.exports.lisspace = lisspace; +module.exports.lisxdigit = lisxdigit; \ No newline at end of file diff --git a/src/llex.js b/src/llex.js index 4c85f65..1adac0a 100644 --- a/src/llex.js +++ b/src/llex.js @@ -4,11 +4,12 @@ const assert = require('assert'); const lapi = require('./lapi.js'); +const lauxlib = require('./lauxlib.js'); const ldebug = require('./ldebug.js'); const ldo = require('./ldo.js'); -const lua = require('./lua.js'); -const lobject = require('./lobject'); const ljstype = require('./ljstype'); +const lobject = require('./lobject'); +const lua = require('./lua.js'); const TValue = lobject.TValue; const CT = lua.constant_types; const TS = lua.thread_status; @@ -24,73 +25,41 @@ const RESERVED = { TK_ELSEIF: FIRST_RESERVED + 4, TK_END: FIRST_RESERVED + 5, TK_FALSE: FIRST_RESERVED + 6, - TK_FOR: FIRST_RESERVED + 8, - TK_FUNCTION: FIRST_RESERVED + 10, - TK_GOTO: FIRST_RESERVED + 11, - TK_IF: FIRST_RESERVED + 12, - TK_IN: FIRST_RESERVED + 13, - TK_LOCAL: FIRST_RESERVED + 14, - TK_NIL: FIRST_RESERVED + 15, - TK_NOT: FIRST_RESERVED + 16, - TK_OR: FIRST_RESERVED + 17, - TK_REPEAT: FIRST_RESERVED + 18, - TK_RETURN: FIRST_RESERVED + 19, - TK_THEN: FIRST_RESERVED + 20, - TK_TRUE: FIRST_RESERVED + 21, - TK_UNTIL: FIRST_RESERVED + 22, - TK_WHILE: FIRST_RESERVED + 23, + TK_FOR: FIRST_RESERVED + 7, + TK_FUNCTION: FIRST_RESERVED + 8, + TK_GOTO: FIRST_RESERVED + 9, + TK_IF: FIRST_RESERVED + 10, + TK_IN: FIRST_RESERVED + 11, + TK_LOCAL: FIRST_RESERVED + 12, + TK_NIL: FIRST_RESERVED + 13, + TK_NOT: FIRST_RESERVED + 14, + TK_OR: FIRST_RESERVED + 15, + TK_REPEAT: FIRST_RESERVED + 16, + TK_RETURN: FIRST_RESERVED + 17, + TK_THEN: FIRST_RESERVED + 18, + TK_TRUE: FIRST_RESERVED + 19, + TK_UNTIL: FIRST_RESERVED + 20, + TK_WHILE: FIRST_RESERVED + 21, /* other terminal symbols */ - TK_IDIV: FIRST_RESERVED + 24, - TK_CONCAT: FIRST_RESERVED + 25, - TK_DOTS: FIRST_RESERVED + 26, - TK_EQ: FIRST_RESERVED + 27, - TK_GE: FIRST_RESERVED + 28, - TK_LE: FIRST_RESERVED + 29, - TK_NE: FIRST_RESERVED + 30, - TK_SHL: FIRST_RESERVED + 31, - TK_SHR: FIRST_RESERVED + 32, - TK_DBCOLON: FIRST_RESERVED + 33, - TK_EOS: FIRST_RESERVED + 34, - TK_FLT: FIRST_RESERVED + 35, - TK_INT: FIRST_RESERVED + 36, - TK_NAME: FIRST_RESERVED + 37, - TK_STRING: FIRST_RESERVED + 38 + TK_IDIV: FIRST_RESERVED + 22, + TK_CONCAT: FIRST_RESERVED + 23, + TK_DOTS: FIRST_RESERVED + 24, + TK_EQ: FIRST_RESERVED + 25, + TK_GE: FIRST_RESERVED + 26, + TK_LE: FIRST_RESERVED + 27, + TK_NE: FIRST_RESERVED + 28, + TK_SHL: FIRST_RESERVED + 29, + TK_SHR: FIRST_RESERVED + 30, + TK_DBCOLON: FIRST_RESERVED + 31, + TK_EOS: FIRST_RESERVED + 32, + TK_FLT: FIRST_RESERVED + 33, + TK_INT: FIRST_RESERVED + 34, + TK_NAME: FIRST_RESERVED + 35, + TK_STRING: FIRST_RESERVED + 36 }; const R = RESERVED; -const reserved_keywords = [ - "and", "break", "do", "else", "elseif", - "end", "false", "for", "function", "goto", "if", - "in", "local", "nil", "not", "or", "repeat", - "return", "then", "true", "until", "while" -]; - -const reserved_keywords_tokens = [ - R.TK_AND, - R.TK_BREAK, - R.TK_DO, - R.TK_ELSE, - R.TK_ELSEIF, - R.TK_END, - R.TK_FALSE, - R.TK_FOR, - R.TK_FUNCTION, - R.TK_GOTO, - R.TK_IF, - R.TK_IN, - R.TK_LOCAL, - R.TK_NIL, - R.TK_NOT, - R.TK_OR, - R.TK_REPEAT, - R.TK_RETURN, - R.TK_THEN, - R.TK_TRUE, - R.TK_UNTIL, - R.TK_WHILE, -]; - const luaX_tokens = [ "and", "break", "do", "else", "elseif", "end", "false", "for", "function", "goto", "if", @@ -103,7 +72,7 @@ const luaX_tokens = [ const NUM_RESERVED = Object.keys(RESERVED).length; -class Buffer { +class MBuffer { constructor(string) { this.buffer = string ? string.split('') : []; this.n = this.buffer.length; @@ -137,8 +106,8 @@ class LexState { this.lookahead = null; /* look ahead token */ this.fs = null; /* current function (parser) */ this.L = null; - this.z = new Buffer(); - this.buff = new Buffer(); /* buffer for tokens */ + this.z = new MBuffer(); + this.buff = new MBuffer(); /* buffer for tokens */ this.h = null; /* to avoid collection/reuse strings */ this.dyd = null; /* dynamic structures used by the parser */ this.source = null; /* current source name */ @@ -195,10 +164,24 @@ const inclinenumber = function(ls) { }; const luaX_setinput = function(L, ls, z, source, firstchar) { - ls.t.token = 0; + ls.t = { + token: 0, + seminfo: { + i: NaN, + r: NaN, + ts: null + } + }; ls.L = L; ls.current = firstchar; - ls.lookahead.token = R.TK_EOS; + ls.lookahead = { + token: R.TK_EOS, + seminfo: { + i: NaN, + r: NaN, + ts: null + } + }; ls.z = z; ls.fs = null; ls.linenumber = 1; @@ -562,8 +545,9 @@ const llex = function(ls, seminfo) { let ts = new TValue(CT.LUA_TLNGSTR, ls.buff.buffer.join('')); seminfo.ts = ts; - if (reserved_keywords.indexOf(ts.value) >= 0) /* reserved word? */ - return reserved_keywords_tokens[reserved_keywords.indexOf(ts.value)]; + let kidx = luaX_tokens.slice(0, 22).indexOf(ts.value) + if (kidx >= 0) /* reserved word? */ + return kidx + FIRST_RESERVED; else return R.TK_NAME; } else { /* single-char tokens (+ - / ...) */ @@ -591,6 +575,11 @@ const luaX_lookahead = function(ls) { return ls.lookahead.token; }; +module.exports.FIRST_RESERVED = FIRST_RESERVED; +module.exports.LexState = LexState; module.exports.luaX_lookahead = luaX_lookahead; module.exports.luaX_next = luaX_next; -module.exports.luaX_setinput = luaX_setinput; \ No newline at end of file +module.exports.luaX_setinput = luaX_setinput; +module.exports.MBuffer = MBuffer; +module.exports.RESERVED = RESERVED; +module.exports.luaX_tokens = luaX_tokens; \ No newline at end of file diff --git a/src/lparser.js b/src/lparser.js new file mode 100644 index 0000000..a82548c --- /dev/null +++ b/src/lparser.js @@ -0,0 +1,15 @@ +/* jshint esversion: 6 */ +"use strict"; + +const assert = require('assert'); + +const llex = require('./llex.js'); +const lfunc = require('./lfunc.js'); + + +const luaY_parser = function(L, z, buff, dyd, name, firstchar) { + // TODO ... +}; + + +module.exports.luaY_parser = luaY_parser; \ No newline at end of file diff --git a/tests/llex.js b/tests/llex.js new file mode 100644 index 0000000..3c05366 --- /dev/null +++ b/tests/llex.js @@ -0,0 +1,50 @@ +/*jshint esversion: 6 */ +"use strict"; + +const test = require('tape'); +const beautify = require('js-beautify').js_beautify; + +const tests = require("./tests.js"); + +const lapi = require("../src/lapi.js"); +const lauxlib = require("../src/lauxlib.js"); +const llex = require("../src/llex.js"); +const lua = require('../src/lua.js'); +const R = llex.RESERVED; + + +test('basic lexing', function (t) { + let luaCode = ` + return "hello lex !" + `, L; + + t.plan(2); + + let readTokens = []; + + t.doesNotThrow(function () { + + L = lauxlib.luaL_newstate(); + + let ls = new llex.LexState(); + llex.luaX_setinput(L, ls, new llex.MBuffer(luaCode), luaCode, luaCode.charAt(0)); + + llex.luaX_next(ls); + + while (ls.t.token !== R.TK_EOS) { + console.log(llex.luaX_tokens[ls.t.token - llex.FIRST_RESERVED]); + + readTokens.push(ls.t.token); + llex.luaX_next(ls); + } + + + }, "JS Lua program ran without error"); + + t.deepEqual( + readTokens, + [R.TK_RETURN, R.TK_STRING], + "Correct tokens found" + ) + +}); \ No newline at end of file -- cgit v1.2.3-70-g09d2