From 29ecfce09eae7e20e06b3937f57c498cd5bd903b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=83=91=E8=8B=8F=E6=B3=A2=20=28Super=20Zheng=29?= Date: Mon, 4 Sep 2023 19:10:37 +0800 Subject: [PATCH 1/2] Fix the WASM Worker cannot start in node.js environment `emscripten_malloc_wasm_worker` and `emscripten_create_wasm_worker` functions adapted to the nodejs environment and modified the core related tests. --- emscripten.py | 1 + src/library_wasi.js | 51 +++++++++++++++++-------- src/library_wasm_worker.js | 13 ++++++- src/parseTools.js | 16 ++++++++ src/preamble.js | 20 ++++------ src/wasm_worker.js | 39 +++++++++++++++++-- test/test_core.py | 11 +++++- test/wasm_worker/hello_wasm_worker.out | 1 + test/wasm_worker/malloc_wasm_worker.out | 1 + 9 files changed, 119 insertions(+), 34 deletions(-) create mode 100644 test/wasm_worker/hello_wasm_worker.out create mode 100644 test/wasm_worker/malloc_wasm_worker.out diff --git a/emscripten.py b/emscripten.py index c2f4977b2ea9d..1380bdcc2b019 100644 --- a/emscripten.py +++ b/emscripten.py @@ -912,6 +912,7 @@ def create_pointer_conversion_wrappers(metadata): '__main_argc_argv': '__PP', 'emscripten_stack_set_limits': '_pp', '__set_stack_limits': '_pp', + '__set_thread_state': '_p___', '__cxa_can_catch': '_ppp', '__cxa_increment_exception_refcount': '_p', '__cxa_decrement_exception_refcount': '_p', diff --git a/src/library_wasi.js b/src/library_wasi.js index 860439fced02d..bc8c80b354e86 100644 --- a/src/library_wasi.js +++ b/src/library_wasi.js @@ -229,7 +229,8 @@ var WasiLibrary = { } return ret; }, -#else +#endif +#if !SYSCALLS_REQUIRE_FILESYSTEM || WASM_WORKERS // MEMFS filesystem disabled lite handling of stdout and stderr: $printCharBuffers: [null, [], []], // 1 => stdout, 2 => stderr $printCharBuffers__internal: true, @@ -247,11 +248,7 @@ var WasiLibrary = { buffer.push(curr); } }, -#endif // SYSCALLS_REQUIRE_FILESYSTEM - -#if SYSCALLS_REQUIRE_FILESYSTEM - fd_write__deps: ['$doWritev'], -#elif (!MINIMAL_RUNTIME || EXIT_RUNTIME) +#if !MINIMAL_RUNTIME || EXIT_RUNTIME $flush_NO_FILESYSTEM__deps: ['$printChar', '$printCharBuffers'], $flush_NO_FILESYSTEM: () => { // flush anything remaining in the buffers during shutdown @@ -261,17 +258,13 @@ var WasiLibrary = { if (printCharBuffers[1].length) printChar(1, {{{ charCode("\n") }}}); if (printCharBuffers[2].length) printChar(2, {{{ charCode("\n") }}}); }, - fd_write__deps: ['$flush_NO_FILESYSTEM', '$printChar'], - fd_write__postset: () => addAtExit('flush_NO_FILESYSTEM()'), + $fd_write_nofs__postset: () => addAtExit('flush_NO_FILESYSTEM()'), + $fd_write_nofs__deps: ['$printChar', '$flush_NO_FILESYSTEM'], #else - fd_write__deps: ['$printChar'], + $fd_write_nofs__deps: ['$printChar'], #endif - fd_write: (fd, iov, iovcnt, pnum) => { -#if SYSCALLS_REQUIRE_FILESYSTEM - var stream = SYSCALLS.getStreamFromFD(fd); - var num = doWritev(stream, iov, iovcnt); -#else - // hack to support printf in SYSCALLS_REQUIRE_FILESYSTEM=0 + $fd_write_nofs__sig: 'iippp', + $fd_write_nofs: (fd, iov, iovcnt, pnum) => { var num = 0; for (var i = 0; i < iovcnt; i++) { var ptr = {{{ makeGetValue('iov', C_STRUCTS.iovec.iov_base, '*') }}}; @@ -282,10 +275,36 @@ var WasiLibrary = { } num += len; } -#endif // SYSCALLS_REQUIRE_FILESYSTEM {{{ makeSetValue('pnum', 0, 'num', SIZE_TYPE) }}}; return 0; }, +#endif + +#if SYSCALLS_REQUIRE_FILESYSTEM + fd_write__deps: [ + '$doWritev', +#if WASM_WORKERS + '$fd_write_nofs', +#endif + ], + fd_write: (fd, iov, iovcnt, pnum) => { +#if WASM_WORKERS + if (ENVIRONMENT_IS_WASM_WORKER) { + return fd_write_nofs(fd, iov, iovcnt, pnum); + } +#endif // WASM_WORKERS + var stream = SYSCALLS.getStreamFromFD(fd); + var num = doWritev(stream, iov, iovcnt); + {{{ makeSetValue('pnum', 0, 'num', SIZE_TYPE) }}}; + return 0; + }, +#else + fd_write__deps: ['$fd_write_nofs'], + fd_write: (fd, iov, iovcnt, pnum) => { + // May be wrapped by wrapSyscallFunction + return fd_write_nofs(fd, iov, iovcnt, pnum); + }, +#endif #if SYSCALLS_REQUIRE_FILESYSTEM fd_pwrite__deps: ['$doWritev'], diff --git a/src/library_wasm_worker.js b/src/library_wasm_worker.js index d1feb29d43d81..3bcba9253d6cb 100644 --- a/src/library_wasm_worker.js +++ b/src/library_wasm_worker.js @@ -47,7 +47,12 @@ addToLibrary({ $_wasmWorkerRunPostMessage: (e) => { // '_wsc' is short for 'wasm call', trying to use an identifier name that // will never conflict with user code - let data = e.data, wasmCall = data['_wsc']; +#if ENVIRONMENT_MAY_BE_NODE + let data = ENVIRONMENT_IS_NODE ? e : e.data; +#else + let data = e.data; +#endif + let wasmCall = data['_wsc']; wasmCall && getWasmTableEntry(wasmCall)(...data['x']); }, @@ -155,6 +160,12 @@ if (ENVIRONMENT_IS_WASM_WORKER) { #endif 'sb': stackLowestAddress, // sb = stack bottom (lowest stack address, SP points at this when stack is full) 'sz': stackSize, // sz = stack size +#if USE_OFFSET_CONVERTER + 'wasmOffsetData': wasmOffsetConverter, +#endif +#if LOAD_SOURCE_MAP + 'wasmSourceMapData': wasmSourceMap, +#endif }); worker.onmessage = _wasmWorkerRunPostMessage; return _wasmWorkersID++; diff --git a/src/parseTools.js b/src/parseTools.js index 9d7296cfbab3c..b4f8f264010ba 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -736,6 +736,18 @@ function runIfMainThread(text) { } } +function runIfWorkerThread(text) { + if (WASM_WORKERS && PTHREADS) { + return 'if (ENVIRONMENT_IS_WASM_WORKER || ENVIRONMENT_IS_PTHREAD) { ' + text + ' }'; + } else if (WASM_WORKERS) { + return 'if (ENVIRONMENT_IS_WASM_WORKER) { ' + text + ' }'; + } else if (PTHREADS) { + return 'if (ENVIRONMENT_IS_PTHREAD) { ' + text + ' }'; + } else { + return ''; + } +} + function expectToReceiveOnModule(name) { return INCOMING_MODULE_JS_API.has(name); } @@ -1031,3 +1043,7 @@ function getPerformanceNow() { return 'performance.now'; } } + +function implicitSelf() { + return ENVIRONMENT.includes('node') ? 'self.' : ''; +} diff --git a/src/preamble.js b/src/preamble.js index 78dc4ce958a1c..24ea0c74d6a3b 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -785,7 +785,7 @@ function instantiateSync(file, info) { } #endif -#if PTHREADS && (LOAD_SOURCE_MAP || USE_OFFSET_CONVERTER) +#if SHARED_MEMORY && (LOAD_SOURCE_MAP || USE_OFFSET_CONVERTER) // When using postMessage to send an object, it is processed by the structured // clone algorithm. The prototype, and hence methods, on that object is then // lost. This function adds back the lost prototype. This does not work with @@ -1034,7 +1034,7 @@ function createWasm() { exportWasmSymbols(wasmExports); #endif -#if PTHREADS || WASM_WORKERS +#if SHARED_MEMORY // We now have the Wasm module loaded up, keep a reference to the compiled module so we can post it to the workers. wasmModule = module; #endif @@ -1082,22 +1082,18 @@ function createWasm() { // path. if (Module['instantiateWasm']) { -#if USE_OFFSET_CONVERTER && PTHREADS - if (ENVIRONMENT_IS_PTHREAD) { +#if USE_OFFSET_CONVERTER #if ASSERTIONS - assert(Module['wasmOffsetData'], 'wasmOffsetData not found on Module object'); +{{{ runIfWorkerThread("assert(Module['wasmOffsetData'], 'wasmOffsetData not found on Module object');") }}} #endif - wasmOffsetConverter = resetPrototype(WasmOffsetConverter, Module['wasmOffsetData']); - } +{{{ runIfWorkerThread("wasmOffsetConverter = resetPrototype(WasmOffsetConverter, Module['wasmOffsetData']);") }}} #endif -#if LOAD_SOURCE_MAP && PTHREADS - if (ENVIRONMENT_IS_PTHREAD) { +#if LOAD_SOURCE_MAP #if ASSERTIONS - assert(Module['wasmSourceMapData'], 'wasmSourceMapData not found on Module object'); +{{{ runIfWorkerThread("assert(Module['wasmSourceMapData'], 'wasmSourceMapData not found on Module object');") }}} #endif - wasmSourceMap = resetPrototype(WasmSourceMap, Module['wasmSourceMapData']); - } +{{{ runIfWorkerThread("wasmSourceMap = resetPrototype(WasmSourceMap, Module['wasmSourceMapData']);") }}} #endif try { diff --git a/src/wasm_worker.js b/src/wasm_worker.js index 3ab43fcbd4b81..8a3507c711c52 100644 --- a/src/wasm_worker.js +++ b/src/wasm_worker.js @@ -1,18 +1,51 @@ // N.B. The contents of this file are duplicated in src/library_wasm_worker.js // in variable "_wasmWorkerBlobUrl" (where the contents are pre-minified) If // doing any changes to this file, be sure to update the contents there too. -onmessage = function(d) { + +'use strict'; + +#if ENVIRONMENT_MAY_BE_NODE +// Node.js support +var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions == 'object' && typeof process.versions.node == 'string'; +if (ENVIRONMENT_IS_NODE) { + // Create as web-worker-like an environment as we can. + + var nodeWorkerThreads = require('worker_threads'); + + var parentPort = nodeWorkerThreads.parentPort; + + parentPort.on('message', (data) => typeof onmessage === "function" && onmessage({ data: data })); + + var fs = require('fs'); + + Object.assign(global, { + self: global, + require, + location: { + href: __filename + }, + Worker: nodeWorkerThreads.Worker, + importScripts: (f) => (0, eval)(fs.readFileSync(f, 'utf8') + '//# sourceURL=' + f), + postMessage: (msg) => parentPort.postMessage(msg), + performance: global.performance || { now: Date.now }, + addEventListener: (name, handler) => parentPort.on(name, handler), + removeEventListener: (name, handler) => parentPort.off(name, handler), + }); +} +#endif // ENVIRONMENT_MAY_BE_NODE + +{{{ implicitSelf() }}}onmessage = function(d) { // The first message sent to the Worker is always the bootstrap message. // Drop this message listener, it served its purpose of bootstrapping // the Wasm Module load, and is no longer needed. Let user code register // any desired message handlers from now on. - onmessage = null; + {{{ implicitSelf() }}}onmessage = null; d = d.data; #if !MODULARIZE self.{{{ EXPORT_NAME }}} = d; #endif #if !MINIMAL_RUNTIME - d['instantiateWasm'] = (info, receiveInstance) => { var instance = new WebAssembly.Instance(d['wasm'], info); receiveInstance(instance, d['wasm']); return instance.exports; } + d['instantiateWasm'] = (info, receiveInstance) => { var instance = new WebAssembly.Instance(d['wasm'], info); return receiveInstance(instance, d['wasm']); } #endif importScripts(d.js); #if MODULARIZE diff --git a/test/test_core.py b/test/test_core.py index 0fc8ab3ee674b..1138c8eba9ec4 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -9912,16 +9912,23 @@ def test_emscripten_async_load_script(self): self.run_process([FILE_PACKAGER, 'test.data', '--preload', 'file1.txt', 'file2.txt', '--from-emcc', '--js-output=script2.js']) self.do_runf(test_file('test_emscripten_async_load_script.c'), emcc_args=['-sFORCE_FILESYSTEM']) + def prep_wasm_worker_in_node(self): + # Auto exit after 3 seconds in Nodejs environment to get WASM Worker stdout + self.add_pre_run("setTimeout(()=>process.exit(), 3000);") + @node_pthreads def test_wasm_worker_hello(self): - self.do_runf(test_file('wasm_worker/hello_wasm_worker.c'), emcc_args=['-sWASM_WORKERS']) + self.prep_wasm_worker_in_node() + self.do_run_in_out_file_test(test_file('wasm_worker/hello_wasm_worker.c'), emcc_args=['-sWASM_WORKERS']) @node_pthreads def test_wasm_worker_malloc(self): - self.do_runf(test_file('wasm_worker/malloc_wasm_worker.c'), emcc_args=['-sWASM_WORKERS']) + self.prep_wasm_worker_in_node() + self.do_run_in_out_file_test(test_file('wasm_worker/malloc_wasm_worker.c'), emcc_args=['-sWASM_WORKERS']) @node_pthreads def test_wasm_worker_wait_async(self): + self.prep_wasm_worker_in_node() self.do_runf(test_file('wasm_worker/wait_async.c'), emcc_args=['-sWASM_WORKERS']) diff --git a/test/wasm_worker/hello_wasm_worker.out b/test/wasm_worker/hello_wasm_worker.out new file mode 100644 index 0000000000000..2b522d9581342 --- /dev/null +++ b/test/wasm_worker/hello_wasm_worker.out @@ -0,0 +1 @@ +Hello from wasm worker! diff --git a/test/wasm_worker/malloc_wasm_worker.out b/test/wasm_worker/malloc_wasm_worker.out new file mode 100644 index 0000000000000..2b522d9581342 --- /dev/null +++ b/test/wasm_worker/malloc_wasm_worker.out @@ -0,0 +1 @@ +Hello from wasm worker! From 55b0af762294e6b0c48f35c290a2aa235a4427f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=83=91=E8=8B=8F=E6=B3=A2=20=28Super=20Zheng=29?= Date: Sat, 7 Oct 2023 15:02:26 +0800 Subject: [PATCH 2/2] Fix the WASM Worker cannot start in node.js environment `emscripten_malloc_wasm_worker` and `emscripten_create_wasm_worker` functions adapted to the nodejs environment and modified the core related tests. --- src/library_wasi.js | 51 +++++++++------------------- src/library_wasm_worker.js | 2 ++ src/parseTools.js | 6 ++-- src/preamble.js | 4 +-- test/wasm_worker/hello_wasm_worker.c | 6 ++-- 5 files changed, 26 insertions(+), 43 deletions(-) diff --git a/src/library_wasi.js b/src/library_wasi.js index bc8c80b354e86..860439fced02d 100644 --- a/src/library_wasi.js +++ b/src/library_wasi.js @@ -229,8 +229,7 @@ var WasiLibrary = { } return ret; }, -#endif -#if !SYSCALLS_REQUIRE_FILESYSTEM || WASM_WORKERS +#else // MEMFS filesystem disabled lite handling of stdout and stderr: $printCharBuffers: [null, [], []], // 1 => stdout, 2 => stderr $printCharBuffers__internal: true, @@ -248,7 +247,11 @@ var WasiLibrary = { buffer.push(curr); } }, -#if !MINIMAL_RUNTIME || EXIT_RUNTIME +#endif // SYSCALLS_REQUIRE_FILESYSTEM + +#if SYSCALLS_REQUIRE_FILESYSTEM + fd_write__deps: ['$doWritev'], +#elif (!MINIMAL_RUNTIME || EXIT_RUNTIME) $flush_NO_FILESYSTEM__deps: ['$printChar', '$printCharBuffers'], $flush_NO_FILESYSTEM: () => { // flush anything remaining in the buffers during shutdown @@ -258,13 +261,17 @@ var WasiLibrary = { if (printCharBuffers[1].length) printChar(1, {{{ charCode("\n") }}}); if (printCharBuffers[2].length) printChar(2, {{{ charCode("\n") }}}); }, - $fd_write_nofs__postset: () => addAtExit('flush_NO_FILESYSTEM()'), - $fd_write_nofs__deps: ['$printChar', '$flush_NO_FILESYSTEM'], + fd_write__deps: ['$flush_NO_FILESYSTEM', '$printChar'], + fd_write__postset: () => addAtExit('flush_NO_FILESYSTEM()'), #else - $fd_write_nofs__deps: ['$printChar'], + fd_write__deps: ['$printChar'], #endif - $fd_write_nofs__sig: 'iippp', - $fd_write_nofs: (fd, iov, iovcnt, pnum) => { + fd_write: (fd, iov, iovcnt, pnum) => { +#if SYSCALLS_REQUIRE_FILESYSTEM + var stream = SYSCALLS.getStreamFromFD(fd); + var num = doWritev(stream, iov, iovcnt); +#else + // hack to support printf in SYSCALLS_REQUIRE_FILESYSTEM=0 var num = 0; for (var i = 0; i < iovcnt; i++) { var ptr = {{{ makeGetValue('iov', C_STRUCTS.iovec.iov_base, '*') }}}; @@ -275,36 +282,10 @@ var WasiLibrary = { } num += len; } +#endif // SYSCALLS_REQUIRE_FILESYSTEM {{{ makeSetValue('pnum', 0, 'num', SIZE_TYPE) }}}; return 0; }, -#endif - -#if SYSCALLS_REQUIRE_FILESYSTEM - fd_write__deps: [ - '$doWritev', -#if WASM_WORKERS - '$fd_write_nofs', -#endif - ], - fd_write: (fd, iov, iovcnt, pnum) => { -#if WASM_WORKERS - if (ENVIRONMENT_IS_WASM_WORKER) { - return fd_write_nofs(fd, iov, iovcnt, pnum); - } -#endif // WASM_WORKERS - var stream = SYSCALLS.getStreamFromFD(fd); - var num = doWritev(stream, iov, iovcnt); - {{{ makeSetValue('pnum', 0, 'num', SIZE_TYPE) }}}; - return 0; - }, -#else - fd_write__deps: ['$fd_write_nofs'], - fd_write: (fd, iov, iovcnt, pnum) => { - // May be wrapped by wrapSyscallFunction - return fd_write_nofs(fd, iov, iovcnt, pnum); - }, -#endif #if SYSCALLS_REQUIRE_FILESYSTEM fd_pwrite__deps: ['$doWritev'], diff --git a/src/library_wasm_worker.js b/src/library_wasm_worker.js index 3bcba9253d6cb..dc45340cd616a 100644 --- a/src/library_wasm_worker.js +++ b/src/library_wasm_worker.js @@ -48,6 +48,8 @@ addToLibrary({ // '_wsc' is short for 'wasm call', trying to use an identifier name that // will never conflict with user code #if ENVIRONMENT_MAY_BE_NODE + // In Node.js environment, message event 'e' containing the actual data sent, + // while in the browser environment it's contained by 'e.data'. let data = ENVIRONMENT_IS_NODE ? e : e.data; #else let data = e.data; diff --git a/src/parseTools.js b/src/parseTools.js index b4f8f264010ba..053d5d7d0b64c 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -738,11 +738,11 @@ function runIfMainThread(text) { function runIfWorkerThread(text) { if (WASM_WORKERS && PTHREADS) { - return 'if (ENVIRONMENT_IS_WASM_WORKER || ENVIRONMENT_IS_PTHREAD) { ' + text + ' }'; + return `if (ENVIRONMENT_IS_WASM_WORKER || ENVIRONMENT_IS_PTHREAD) { ${text} }`; } else if (WASM_WORKERS) { - return 'if (ENVIRONMENT_IS_WASM_WORKER) { ' + text + ' }'; + return `if (ENVIRONMENT_IS_WASM_WORKER) { ${text} }`; } else if (PTHREADS) { - return 'if (ENVIRONMENT_IS_PTHREAD) { ' + text + ' }'; + return `if (ENVIRONMENT_IS_PTHREAD) { ${text} }`; } else { return ''; } diff --git a/src/preamble.js b/src/preamble.js index 24ea0c74d6a3b..a368db2c43615 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -785,7 +785,7 @@ function instantiateSync(file, info) { } #endif -#if SHARED_MEMORY && (LOAD_SOURCE_MAP || USE_OFFSET_CONVERTER) +#if (PTHREADS || WASM_WORKERS) && (LOAD_SOURCE_MAP || USE_OFFSET_CONVERTER) // When using postMessage to send an object, it is processed by the structured // clone algorithm. The prototype, and hence methods, on that object is then // lost. This function adds back the lost prototype. This does not work with @@ -1034,7 +1034,7 @@ function createWasm() { exportWasmSymbols(wasmExports); #endif -#if SHARED_MEMORY +#if PTHREADS || WASM_WORKERS // We now have the Wasm module loaded up, keep a reference to the compiled module so we can post it to the workers. wasmModule = module; #endif diff --git a/test/wasm_worker/hello_wasm_worker.c b/test/wasm_worker/hello_wasm_worker.c index a0047c0d0af1c..95abab5db2cab 100644 --- a/test/wasm_worker/hello_wasm_worker.c +++ b/test/wasm_worker/hello_wasm_worker.c @@ -1,12 +1,12 @@ -#include +#include #include -#include +#include // This is the code example in site/source/docs/api_reference/wasm_workers.rst void run_in_worker() { - printf("Hello from wasm worker!\n"); + emscripten_console_log("Hello from wasm worker!\n"); #ifdef REPORT_RESULT REPORT_RESULT(0); #endif