[test-suite] utf8.js passes

3 files changed, 400 insertions, 4 deletions

diff --git a/README.md b/README.md
index ee8eb81..f5ee3ac 100644
--- a/README.md
+++ b/README.md
@@ -69,10 +69,10 @@
     - [x] `nextvar.lua` (44/44)
     - [x] `sort.lua` (24/24)
     - [x] `strings.lua` (34/34)
+    - [x] `utf8.lua` (18/18)
     - [x] `vararg.lua` (8/8)
     - [ ] `pm.lua` (27/38)
     - [ ] `tpack.lua` (20/32)
-    - [ ] `utf8.lua` (14/20)
     - [ ] `attrib.lua`
     - [ ] `big.lua`
     - [ ] `db.lua`
diff --git a/tests/test-suite/inprogress/utf8.js b/tests/test-suite/inprogress/utf8.js
index e3f63d1..2da2d4a 100644
--- a/tests/test-suite/inprogress/utf8.js
+++ b/tests/test-suite/inprogress/utf8.js
@@ -4,9 +4,9 @@ const test     = require('tape');
 
 global.WEB = false;
 
-const lua     = require('../../../src/lua.js');
-const lauxlib = require('../../../src/lauxlib.js');
-const lualib  = require('../../../src/lualib.js');
+const lua     = require('../../src/lua.js');
+const lauxlib = require('../../src/lauxlib.js');
+const lualib  = require('../../src/lualib.js');
 
 
 const prefix = `
diff --git a/tests/test-suite/utf8.js b/tests/test-suite/utf8.js
new file mode 100644
index 0000000..2da2d4a
--- /dev/null
+++ b/tests/test-suite/utf8.js
@@ -0,0 +1,396 @@
+"use strict";
+
+const test     = require('tape');
+
+global.WEB = false;
+
+const lua     = require('../../src/lua.js');
+const lauxlib = require('../../src/lauxlib.js');
+const lualib  = require('../../src/lualib.js');
+
+
+const prefix = `
+    local function checkerror (msg, f, ...)
+      local s, err = pcall(f, ...)
+      assert(not s and string.find(err, msg))
+    end
+
+
+    local function len (s)
+      return #string.gsub(s, "[\\x80-\\xBF]", "")
+    end
+
+
+    local justone = "^" .. utf8.charpattern .. "$"
+
+    local function checksyntax (s, t)
+      local ts = {"return '"}
+      for i = 1, #t do ts[i + 1] = string.format("\\\\u{%x}", t[i]) end
+      ts[#t + 2] = "'"
+      ts = table.concat(ts)
+      assert(assert(load(ts))() == s)
+    end
+    -- 't' is the list of codepoints of 's'
+    local function check (s, t)
+      local l = utf8.len(s) 
+      assert(#t == l and len(s) == l)
+      assert(utf8.char(table.unpack(t)) == s)
+
+      assert(utf8.offset(s, 0) == 1)
+
+      checksyntax(s, t)
+
+      local t1 = {utf8.codepoint(s, 1, -1)}
+      assert(#t == #t1)
+      for i = 1, #t do assert(t[i] == t1[i]) end
+
+      for i = 1, l do
+        local pi = utf8.offset(s, i)        -- position of i-th char
+        local pi1 = utf8.offset(s, 2, pi)   -- position of next char
+        assert(string.find(string.sub(s, pi, pi1 - 1), justone))
+        assert(utf8.offset(s, -1, pi1) == pi)
+        assert(utf8.offset(s, i - l - 1) == pi)
+        assert(pi1 - pi == #utf8.char(utf8.codepoint(s, pi)))
+        for j = pi, pi1 - 1 do 
+          assert(utf8.offset(s, 0, j) == pi)
+        end
+        for j = pi + 1, pi1 - 1 do
+          assert(not utf8.len(s, j))
+        end
+       assert(utf8.len(s, pi, pi) == 1)
+       assert(utf8.len(s, pi, pi1 - 1) == 1)
+       assert(utf8.len(s, pi) == l - i + 1)
+       assert(utf8.len(s, pi1) == l - i)
+       assert(utf8.len(s, 1, pi) == i)
+      end
+
+      local i = 0
+      for p, c in utf8.codes(s) do
+        i = i + 1
+        assert(c == t[i] and p == utf8.offset(s, i))
+        assert(utf8.codepoint(s, p) == c)
+      end
+      assert(i == #t)
+
+      i = 0
+      for p, c in utf8.codes(s) do
+        i = i + 1
+        assert(c == t[i] and p == utf8.offset(s, i)) 
+      end
+      assert(i == #t)
+
+      i = 0
+      for c in string.gmatch(s, utf8.charpattern) do
+        i = i + 1
+        assert(c == utf8.char(t[i]))
+      end
+      assert(i == #t)
+
+      for i = 1, l do
+        assert(utf8.offset(s, i) == utf8.offset(s, i - l - 1, #s + 1))
+      end
+
+    end
+
+    local function invalid (s)
+      checkerror("invalid UTF%-8 code", utf8.codepoint, s)
+      assert(not utf8.len(s))
+    end
+`;
+
+test("[test-suite] utf8: offset", function (t) {
+    let luaCode = `
+        assert(utf8.offset("alo", 5) == nil)
+        assert(utf8.offset("alo", -4) == nil)
+    `, L;
+    
+    t.plan(2);
+
+    t.doesNotThrow(function () {
+
+        L = lauxlib.luaL_newstate();
+
+        lualib.luaL_openlibs(L);
+
+        lauxlib.luaL_loadstring(L, lua.to_luastring(prefix + luaCode));
+
+    }, "Lua program loaded without error");
+
+    t.doesNotThrow(function () {
+
+        lua.lua_call(L, 0, -1);
+
+    }, "Lua program ran without error");
+});
+
+
+test("[test-suite] utf8: error indication in utf8.len", function (t) {
+    let luaCode = `
+        do
+          local function check (s, p)
+            local a, b = utf8.len(s)
+            assert(not a and b == p)
+          end
+          check("abc\\xE3def", 4)
+          check("汉字\\x80", #("汉字") + 1)
+          check("\\xF4\\x9F\\xBF", 1)
+          check("\\xF4\\x9F\\xBF\\xBF", 1)
+        end
+    `, L;
+    
+    t.plan(2);
+
+    t.doesNotThrow(function () {
+
+        L = lauxlib.luaL_newstate();
+
+        lualib.luaL_openlibs(L);
+
+        lauxlib.luaL_loadstring(L, lua.to_luastring(prefix + luaCode));
+
+    }, "Lua program loaded without error");
+
+    t.doesNotThrow(function () {
+
+        lua.lua_call(L, 0, -1);
+
+    }, "Lua program ran without error");
+});
+
+
+test("[test-suite] utf8: error in initial position for offset", function (t) {
+    let luaCode = `
+        checkerror("position out of range", utf8.offset, "abc", 1, 5)
+        checkerror("position out of range", utf8.offset, "abc", 1, -4)
+        checkerror("position out of range", utf8.offset, "", 1, 2)
+        checkerror("position out of range", utf8.offset, "", 1, -1)
+        checkerror("continuation byte", utf8.offset, "𦧺", 1, 2)
+        checkerror("continuation byte", utf8.offset, "𦧺", 1, 2)
+        checkerror("continuation byte", utf8.offset, "\\x80", 1)
+    `, L;
+    
+    t.plan(2);
+
+    t.doesNotThrow(function () {
+
+        L = lauxlib.luaL_newstate();
+
+        lualib.luaL_openlibs(L);
+
+        lauxlib.luaL_loadstring(L, lua.to_luastring(prefix + luaCode));
+
+    }, "Lua program loaded without error");
+
+    t.doesNotThrow(function () {
+
+        lua.lua_call(L, 0, -1);
+
+    }, "Lua program ran without error");
+});
+
+
+test("[test-suite] utf8: codepoints", function (t) {
+    let luaCode = `
+        local s = "hello World"
+        local t = {string.byte(s, 1, -1)}
+        for i = 1, utf8.len(s) do assert(t[i] == string.byte(s, i)) end
+        check(s, t)
+
+        check("汉字/漢字", {27721, 23383, 47, 28450, 23383,})
+
+        do
+          local s = "áéí\\128"
+          local t = {utf8.codepoint(s,1,#s - 1)}
+          assert(#t == 3 and t[1] == 225 and t[2] == 233 and t[3] == 237)
+          checkerror("invalid UTF%-8 code", utf8.codepoint, s, 1, #s)
+          checkerror("out of range", utf8.codepoint, s, #s + 1)
+          t = {utf8.codepoint(s, 4, 3)}
+          assert(#t == 0)
+          checkerror("out of range", utf8.codepoint, s, -(#s + 1), 1)
+          checkerror("out of range", utf8.codepoint, s, 1, #s + 1)
+        end
+
+        assert(utf8.char() == "")
+        assert(utf8.char(97, 98, 99) == "abc")
+
+        assert(utf8.codepoint(utf8.char(0x10FFFF)) == 0x10FFFF)
+
+        checkerror("value out of range", utf8.char, 0x10FFFF + 1)
+    `, L;
+    
+    t.plan(2);
+
+    t.doesNotThrow(function () {
+
+        L = lauxlib.luaL_newstate();
+
+        lualib.luaL_openlibs(L);
+
+        lauxlib.luaL_loadstring(L, lua.to_luastring(prefix + luaCode));
+
+    }, "Lua program loaded without error");
+
+    t.doesNotThrow(function () {
+
+        lua.lua_call(L, 0, -1);
+
+    }, "Lua program ran without error");
+});
+
+
+test("[test-suite] utf8: UTF-8 representation for 0x11ffff (value out of valid range)", function (t) {
+    let luaCode = `
+        invalid("\\xF4\\x9F\\xBF\\xBF")
+    `, L;
+    
+    t.plan(2);
+
+    t.doesNotThrow(function () {
+
+        L = lauxlib.luaL_newstate();
+
+        lualib.luaL_openlibs(L);
+
+        lauxlib.luaL_loadstring(L, lua.to_luastring(prefix + luaCode));
+
+    }, "Lua program loaded without error");
+
+    t.doesNotThrow(function () {
+
+        lua.lua_call(L, 0, -1);
+
+    }, "Lua program ran without error");
+});
+
+
+test("[test-suite] utf8: overlong sequences", function (t) {
+    let luaCode = `
+        invalid("\\xC0\\x80")          -- zero
+        invalid("\\xC1\\xBF")          -- 0x7F (should be coded in 1 byte)
+        invalid("\\xE0\\x9F\\xBF")      -- 0x7FF (should be coded in 2 bytes)
+        invalid("\\xF0\\x8F\\xBF\\xBF")  -- 0xFFFF (should be coded in 3 bytes)
+    `, L;
+    
+    t.plan(2);
+
+    t.doesNotThrow(function () {
+
+        L = lauxlib.luaL_newstate();
+
+        lualib.luaL_openlibs(L);
+
+        lauxlib.luaL_loadstring(L, lua.to_luastring(prefix + luaCode));
+
+    }, "Lua program loaded without error");
+
+    t.doesNotThrow(function () {
+
+        lua.lua_call(L, 0, -1);
+
+    }, "Lua program ran without error");
+});
+
+
+test("[test-suite] utf8: invalid bytes", function (t) {
+    let luaCode = `
+        invalid("\\x80")  -- continuation byte
+        invalid("\\xBF")  -- continuation byte
+        invalid("\\xFE")  -- invalid byte
+        invalid("\\xFF")  -- invalid byte
+    `, L;
+    
+    t.plan(2);
+
+    t.doesNotThrow(function () {
+
+        L = lauxlib.luaL_newstate();
+
+        lualib.luaL_openlibs(L);
+
+        lauxlib.luaL_loadstring(L, lua.to_luastring(prefix + luaCode));
+
+    }, "Lua program loaded without error");
+
+    t.doesNotThrow(function () {
+
+        lua.lua_call(L, 0, -1);
+
+    }, "Lua program ran without error");
+});
+
+
+test("[test-suite] utf8: empty strings", function (t) {
+    let luaCode = `
+        check("", {})
+    `, L;
+    
+    t.plan(2);
+
+    t.doesNotThrow(function () {
+
+        L = lauxlib.luaL_newstate();
+
+        lualib.luaL_openlibs(L);
+
+        lauxlib.luaL_loadstring(L, lua.to_luastring(prefix + luaCode));
+
+    }, "Lua program loaded without error");
+
+    t.doesNotThrow(function () {
+
+        lua.lua_call(L, 0, -1);
+
+    }, "Lua program ran without error");
+});
+
+
+test("[test-suite] utf8: minimum and maximum values for each sequence size", function (t) {
+    let luaCode = `
+        s = "\\0 \\x7F\\z
+             \\xC2\\x80 \\xDF\\xBF\\z
+             \\xE0\\xA0\\x80 \\xEF\\xBF\\xBF\\z
+             \\xF0\\x90\\x80\\x80  \\xF4\\x8F\\xBF\\xBF"
+        s = string.gsub(s, " ", "")
+        check(s, {0,0x7F, 0x80,0x7FF, 0x800,0xFFFF, 0x10000,0x10FFFF})
+
+        x = "日本語a-4\\0éó"
+        check(x, {26085, 26412, 35486, 97, 45, 52, 0, 233, 243})
+
+        -- Supplementary Characters
+        check("𣲷𠜎𠱓𡁻𠵼ab𠺢",
+              {0x23CB7, 0x2070E, 0x20C53, 0x2107B, 0x20D7C, 0x61, 0x62, 0x20EA2,})
+
+        check("𨳊𩶘𦧺𨳒𥄫𤓓\\xF4\\x8F\\xBF\\xBF",
+              {0x28CCA, 0x29D98, 0x269FA, 0x28CD2, 0x2512B, 0x244D3, 0x10ffff})
+
+
+        local i = 0
+        for p, c in string.gmatch(x, "()(" .. utf8.charpattern .. ")") do
+          i = i + 1
+          assert(utf8.offset(x, i) == p)
+          assert(utf8.len(x, p) == utf8.len(x) - i + 1)
+          assert(utf8.len(c) == 1)
+          for j = 1, #c - 1 do
+            assert(utf8.offset(x, 0, p + j - 1) == p)
+          end
+        end
+    `, L;
+
+    t.plan(2);
+
+    t.doesNotThrow(function () {
+
+        L = lauxlib.luaL_newstate();
+
+        lualib.luaL_openlibs(L);
+
+        lauxlib.luaL_loadstring(L, lua.to_luastring(prefix + luaCode));
+
+    }, "Lua program loaded without error");
+
+    t.doesNotThrow(function () {
+
+        lua.lua_call(L, 0, -1);
+
+    }, "Lua program ran without error");
+});