binary data support

build.sh

@@ -24,6 +24,9 @@ emcc \
         'setValue', \
         'lengthBytesUTF8', \
         'stringToUTF8', \
+        'HEAPU32', \
+        'HEAPU8', \
+        'HEAP8', \
         'stringToNewUTF8'
     ]" \
     -s INCOMING_MODULE_JS_API="[

package.json

@@ -60,6 +60,7 @@
         "typescript-eslint": "8.18.2"
     },
     "dependencies": {
-        "@types/emscripten": "1.39.10"
+        "@types/emscripten": "1.39.10",
+        "isutf8": "^4.0.1"
     }
 }

src/engine.ts

@@ -9,6 +9,8 @@ import createPromiseType from './type-extensions/promise'
 import createProxyType from './type-extensions/proxy'
 import createTableType from './type-extensions/table'
 import createUserdataType from './type-extensions/userdata'
+import createBinaryStringType from './type-extensions/binary-string'
+import createStringType from './type-extensions/string'
 
 export default class LuaEngine {
     public global: Global
@@ -21,10 +23,19 @@ export default class LuaEngine {
             enableProxy = true,
             traceAllocations = false,
             functionTimeout = undefined as number | undefined,
+            binaryString = false,
         }: CreateEngineOptions = {},
     ) {
         this.global = new Global(this.cmodule, traceAllocations)
 
+        // This should be high priority since it is a primitive type.
+        this.global.registerTypeExtension(6, createStringType(this.global))
+
+        if (binaryString) {
+            // This should be higher priority since it is an override for primitive type
+            this.global.registerTypeExtension(7, createBinaryStringType(this.global))
+        }
+
         // Generic handlers - These may be required to be registered for additional types.
         this.global.registerTypeExtension(0, createTableType(this.global))
         this.global.registerTypeExtension(0, createFunctionType(this.global, { functionTimeout }))

src/luawasm.ts

@@ -111,6 +111,7 @@ export default class LuaWasm {
     public lua_tointegerx: (L: LuaState, idx: number, isnum: number | null) => bigint
     public lua_toboolean: (L: LuaState, idx: number) => number
     public lua_tolstring: (L: LuaState, idx: number, len: number | null) => string
+    public lua_ptr_tolstring: (L: LuaState, idx: number, len: number | null) => number
     public lua_rawlen: (L: LuaState, idx: number) => number
     public lua_tocfunction: (L: LuaState, idx: number) => number
     public lua_touserdata: (L: LuaState, idx: number) => number
@@ -268,6 +269,7 @@ export default class LuaWasm {
         this.lua_tointegerx = this.cwrap('lua_tointegerx', 'number', ['number', 'number', 'number'])
         this.lua_toboolean = this.cwrap('lua_toboolean', 'number', ['number', 'number'])
         this.lua_tolstring = this.cwrap('lua_tolstring', 'string', ['number', 'number', 'number'])
+        this.lua_ptr_tolstring = this.cwrap('lua_tolstring', 'number', ['number', 'number', 'number'])
         this.lua_rawlen = this.cwrap('lua_rawlen', 'number', ['number', 'number'])
         this.lua_tocfunction = this.cwrap('lua_tocfunction', 'number', ['number', 'number'])
         this.lua_touserdata = this.cwrap('lua_touserdata', 'number', ['number', 'number'])

src/thread.ts

@@ -210,9 +210,6 @@ export default class Thread {
                     this.lua.lua_pushnumber(this.address, target)
                 }
                 break
-            case 'string':
-                this.lua.lua_pushstring(this.address, target)
-                break
             case 'boolean':
                 this.lua.lua_pushboolean(this.address, target ? 1 : 0)
                 break
@@ -280,8 +277,6 @@ export default class Thread {
                 return null
             case LuaType.Number:
                 return this.lua.lua_tonumberx(this.address, index, null)
-            case LuaType.String:
-                return this.lua.lua_tolstring(this.address, index, null)
             case LuaType.Boolean:
                 return Boolean(this.lua.lua_toboolean(this.address, index))
             case LuaType.Thread:

src/type-extensions/binary-string.ts

@@ -0,0 +1,59 @@
+import LuaTypeExtension from '../type-extension'
+import Global from '../global'
+import Thread from '../thread'
+import { Decoration } from '../decoration'
+import { LuaType, PointerSize } from '../types'
+import isUtf8 from 'isutf8'
+
+type BinaryType = Uint8Array<ArrayBufferLike> | string
+
+class BinaryStringExtension extends LuaTypeExtension<BinaryType> {
+    constructor(thread: Global) {
+        super(thread, 'js_unit8array')
+    }
+
+    public pushValue(thread: Thread, { target }: Decoration<Uint8Array>): boolean {
+        if (target instanceof Uint8Array) {
+            thread.lua.lua_checkstack(thread.address, 1)
+            const bufferSize = target.byteLength
+            const bufferPtr = thread.lua.module._malloc(bufferSize)
+
+            thread.lua.module.HEAP8.set(target, bufferPtr)
+            thread.lua.lua_pushlstring(thread.address, bufferPtr, bufferSize)
+            thread.lua.module._free(bufferPtr)
+
+            return true
+        }
+
+        return false
+    }
+
+    public isType(_thread: Thread, _index: number, type: LuaType): boolean {
+        return type === LuaType.String
+    }
+
+    public getValue(thread: Thread, index: number, _userdata?: unknown): BinaryType {
+        const lenPtr = thread.lua.module._malloc(PointerSize)
+        const bufferPtr = thread.lua.lua_ptr_tolstring(thread.address, index, lenPtr)
+        const length = thread.lua.module.HEAPU32[lenPtr / Uint32Array.BYTES_PER_ELEMENT]
+        thread.lua.module._free(lenPtr)
+
+        const dataView = thread.lua.module.HEAPU8.subarray(bufferPtr, bufferPtr + length)
+
+        if (isUtf8(dataView)) {
+            const decoder = new TextDecoder('utf-8')
+            const decodedString = decoder.decode(dataView)
+            return decodedString
+        }
+
+        return dataView
+    }
+
+    public close(): void {
+        // Nothing to do
+    }
+}
+
+export default function createTypeExtension(thread: Global): LuaTypeExtension<BinaryType> {
+    return new BinaryStringExtension(thread)
+}

src/type-extensions/string.ts

@@ -0,0 +1,35 @@
+import LuaTypeExtension from '../type-extension'
+import Global from '../global'
+import { LuaType } from '../types'
+import Thread from '../thread'
+
+class BasicStringExtension extends LuaTypeExtension<string> {
+    public constructor(thread: Global) {
+        super(thread, 'js_string')
+    }
+
+    public pushValue(thread: Thread, { target }: { target: unknown }): boolean {
+        if (typeof target !== 'string') {
+            return false
+        }
+
+        thread.lua.lua_pushstring(thread.address, target)
+        return true
+    }
+
+    public isType(_thread: Thread, _index: number, type: number): boolean {
+        return type === LuaType.String
+    }
+
+    public getValue(thread: Thread, index: number): string {
+        return thread.lua.lua_tolstring(thread.address, index, null)
+    }
+
+    public close(): void {
+        // Nothing to do
+    }
+}
+
+export default function createTypeExtension(thread: Global): LuaTypeExtension<string> {
+    return new BasicStringExtension(thread)
+}

src/types.ts

@@ -13,6 +13,8 @@ export interface CreateEngineOptions {
     traceAllocations?: boolean
     /** Maximum time in milliseconds a Lua function can run before being interrupted. */
     functionTimeout?: number
+    /** Whether to target lua string as binary data in general, not only UTF encoding, and support byte array as input */
+    binaryString?: boolean
 }
 
 export enum LuaReturn {

test/binary.test.js

@@ -0,0 +1,37 @@
+import { expect } from 'chai'
+import { getEngine } from './utils.js'
+
+describe('Binary', () => {
+    it('passes utf8 buffer correctly', async () => {
+        const engine = await getEngine({ binaryString: true })
+        const b = Buffer.from('мой тёст', 'utf8')
+        engine.global.set('str', b)
+
+        const res = await engine.doString('return tostring(str == "мой тёст")')
+
+        expect(res).to.be.equal('true')
+    })
+
+    it('passes utf8 string correctly', async () => {
+        const engine = await getEngine()
+        engine.global.set('str', 'мой тёст')
+
+        const res = await engine.doString('return tostring(str == "мой тёст")')
+
+        expect(res).to.be.equal('true')
+    })
+
+    it('passes binary data correctly', async () => {
+        // 1px png image
+        const img = Buffer.from(
+            '89504e470d0a1a0a0000000d4948445200000001000000010802000000907753de0000001974455874536f6674776172650041646f626520496d616765526561647971c9653c0000032669545874584d4c3a636f6d2e61646f62652e786d7000000000003c3f787061636b657420626567696e3d22efbbbf222069643d2257354d304d7043656869487a7265537a4e54637a6b633964223f3e203c783a786d706d65746120786d6c6e733a783d2261646f62653a6e733a6d6574612f2220783a786d70746b3d2241646f626520584d5020436f726520352e362d633133382037392e3135393832342c20323031362f30392f31342d30313a30393a30312020202020202020223e203c7264663a52444620786d6c6e733a7264663d22687474703a2f2f7777772e77332e6f72672f313939392f30322f32322d7264662d73796e7461782d6e7323223e203c7264663a4465736372697074696f6e207264663a61626f75743d222220786d6c6e733a786d703d22687474703a2f2f6e732e61646f62652e636f6d2f7861702f312e302f2220786d6c6e733a786d704d4d3d22687474703a2f2f6e732e61646f62652e636f6d2f7861702f312e302f6d6d2f2220786d6c6e733a73745265663d22687474703a2f2f6e732e61646f62652e636f6d2f7861702f312e302f73547970652f5265736f75726365526566232220786d703a43726561746f72546f6f6c3d2241646f62652050686f746f73686f702043432032303137202857696e646f7773292220786d704d4d3a496e7374616e636549443d22786d702e6969643a46323542393237393031313031314538393731334432373436344437393337362220786d704d4d3a446f63756d656e7449443d22786d702e6469643a4632354239323741303131303131453839373133443237343634443739333736223e203c786d704d4d3a4465726976656446726f6d2073745265663a696e7374616e636549443d22786d702e6969643a4632354239323737303131303131453839373133443237343634443739333736222073745265663a646f63756d656e7449443d22786d702e6469643a4632354239323738303131303131453839373133443237343634443739333736222f3e203c2f7264663a4465736372697074696f6e3e203c2f7264663a5244463e203c2f783a786d706d6574613e203c3f787061636b657420656e643d2272223f3e8503272a0000000f4944415478da62faffff3f40800100060603002b8974ad0000000049454e44ae426082',
+            'hex',
+        )
+        const engine = await getEngine({ binaryString: true })
+        engine.global.set('img', img)
+
+        const res = await engine.doString('return img')
+
+        expect(Buffer.from(res).toString('hex')).to.be.equal(img.toString('hex'))
+    })
+})

test/engine.test.js

@@ -806,13 +806,13 @@ describe('Engine', () => {
 
     it('lots of doString calls should succeed', async () => {
         const engine = await getEngine()
-        const length = 10000;
+        const length = 10000
 
         for (let i = 0; i < length; i++) {
-            const a = Math.floor(Math.random() * 100);
-            const b = Math.floor(Math.random() * 100);
-            const result = await engine.doString(`return ${a} + ${b};`);
-            expect(result).to.equal(a + b);
+            const a = Math.floor(Math.random() * 100)
+            const b = Math.floor(Math.random() * 100)
+            const result = await engine.doString(`return ${a} + ${b};`)
+            expect(result).to.equal(a + b)
         }
     })
 })