Fix the WASM Worker cannot start in node.js environment

`emscripten_malloc_wasm_worker` and
`emscripten_create_wasm_worker` functions
adapted to the nodejs environment and
modified the core related tests.

Author: medns

emscripten.py

@@ -912,6 +912,7 @@ def create_pointer_conversion_wrappers(metadata):
     '__main_argc_argv': '__PP',
     'emscripten_stack_set_limits': '_pp',
     '__set_stack_limits': '_pp',
+    '__set_thread_state': '_p___',
     '__cxa_can_catch': '_ppp',
     '__cxa_increment_exception_refcount': '_p',
     '__cxa_decrement_exception_refcount': '_p',

src/library_wasi.js

@@ -229,7 +229,8 @@ var WasiLibrary = {
     }
     return ret;
   },
-#else
+#endif
+#if !SYSCALLS_REQUIRE_FILESYSTEM || WASM_WORKERS
   // MEMFS filesystem disabled lite handling of stdout and stderr:
   $printCharBuffers: [null, [], []], // 1 => stdout, 2 => stderr
   $printCharBuffers__internal: true,
@@ -247,11 +248,9 @@ var WasiLibrary = {
       buffer.push(curr);
     }
   },
-#endif // SYSCALLS_REQUIRE_FILESYSTEM
+#endif
 
-#if SYSCALLS_REQUIRE_FILESYSTEM
-  fd_write__deps: ['$doWritev'],
-#elif (!MINIMAL_RUNTIME || EXIT_RUNTIME)
+#if ((!SYSCALLS_REQUIRE_FILESYSTEM || WASM_WORKERS) && (!MINIMAL_RUNTIME || EXIT_RUNTIME))
   $flush_NO_FILESYSTEM__deps: ['$printChar', '$printCharBuffers'],
   $flush_NO_FILESYSTEM: () => {
     // flush anything remaining in the buffers during shutdown
@@ -261,17 +260,41 @@ var WasiLibrary = {
     if (printCharBuffers[1].length) printChar(1, {{{ charCode("\n") }}});
     if (printCharBuffers[2].length) printChar(2, {{{ charCode("\n") }}});
   },
-  fd_write__deps: ['$flush_NO_FILESYSTEM', '$printChar'],
-  fd_write__postset: () => addAtExit('flush_NO_FILESYSTEM()'),
+#if SYSCALLS_REQUIRE_FILESYSTEM
+  fd_write__postset: () => addAtExit('if(ENVIRONMENT_IS_WASM_WORKER) flush_NO_FILESYSTEM()'),
 #else
-  fd_write__deps: ['$printChar'],
+  fd_write__postset: () => addAtExit('flush_NO_FILESYSTEM()'),
+#endif
+#endif
+
+  fd_write__deps: [
+#if SYSCALLS_REQUIRE_FILESYSTEM
+    '$doWritev',
+#endif
+#if (!SYSCALLS_REQUIRE_FILESYSTEM || WASM_WORKERS)
+    '$printChar',
+#if (!MINIMAL_RUNTIME || EXIT_RUNTIME)
+    '$flush_NO_FILESYSTEM',
 #endif
+#endif
+],
   fd_write: (fd, iov, iovcnt, pnum) => {
 #if SYSCALLS_REQUIRE_FILESYSTEM
+#if WASM_WORKERS
+  if (!ENVIRONMENT_IS_WASM_WORKER) {
+#endif // WASM_WORKERS
     var stream = SYSCALLS.getStreamFromFD(fd);
     var num = doWritev(stream, iov, iovcnt);
-#else
-    // hack to support printf in SYSCALLS_REQUIRE_FILESYSTEM=0
+#if WASM_WORKERS
+  } else {
+    // Hack to support printf in WASM workers in a simple way.
+    // We support at least stdout, stderr.
+#endif // WASM_WORKERS
+#endif // SYSCALLS_REQUIRE_FILESYSTEM
+#if !SYSCALLS_REQUIRE_FILESYSTEM || WASM_WORKERS
+#if !SYSCALLS_REQUIRE_FILESYSTEM
+    // Hack to support printf in SYSCALLS_REQUIRE_FILESYSTEM=0
+#endif
     var num = 0;
     for (var i = 0; i < iovcnt; i++) {
       var ptr = {{{ makeGetValue('iov', C_STRUCTS.iovec.iov_base, '*') }}};
@@ -282,7 +305,10 @@ var WasiLibrary = {
       }
       num += len;
     }
-#endif // SYSCALLS_REQUIRE_FILESYSTEM
+#endif
+#if SYSCALLS_REQUIRE_FILESYSTEM && WASM_WORKERS
+  }
+#endif
     {{{ makeSetValue('pnum', 0, 'num', SIZE_TYPE) }}};
     return 0;
   },

src/library_wasm_worker.js

@@ -47,7 +47,12 @@ addToLibrary({
   $_wasmWorkerRunPostMessage: (e) => {
     // '_wsc' is short for 'wasm call', trying to use an identifier name that
     // will never conflict with user code
-    let data = e.data, wasmCall = data['_wsc'];
+#if ENVIRONMENT_MAY_BE_NODE
+    let data = ENVIRONMENT_IS_NODE ? e : e.data;
+#else
+    let data = e.data;
+#endif
+    let wasmCall = data['_wsc'];
     wasmCall && getWasmTableEntry(wasmCall)(...data['x']);
   },
 
@@ -155,6 +160,12 @@ if (ENVIRONMENT_IS_WASM_WORKER) {
 #endif
       'sb': stackLowestAddress, // sb = stack bottom (lowest stack address, SP points at this when stack is full)
       'sz': stackSize,          // sz = stack size
+#if USE_OFFSET_CONVERTER
+      'wasmOffsetData': wasmOffsetConverter,
+#endif
+#if LOAD_SOURCE_MAP
+      'wasmSourceMapData': wasmSourceMap,
+#endif
     });
     worker.onmessage = _wasmWorkerRunPostMessage;
     return _wasmWorkersID++;

src/parseTools.js

@@ -736,6 +736,18 @@ function runIfMainThread(text) {
   }
 }
 
+function runIfWorkerThread(text) {
+  if (WASM_WORKERS && PTHREADS) {
+    return 'if (ENVIRONMENT_IS_WASM_WORKER || ENVIRONMENT_IS_PTHREAD) { ' + text + ' }';
+  } else if (WASM_WORKERS) {
+    return 'if (ENVIRONMENT_IS_WASM_WORKER) { ' + text + ' }';
+  } else if (PTHREADS) {
+    return 'if (ENVIRONMENT_IS_PTHREAD) { ' + text + ' }';
+  } else {
+    return '';
+  }
+}
+
 // Legacy name for runIfMainThread.
 // TODO(remove).
 const runOnMainThread = runIfMainThread;
@@ -1023,3 +1035,7 @@ function getPerformanceNow() {
     return 'performance.now';
   }
 }
+
+function implicitSelf() {
+  return ENVIRONMENT.includes('node') ? 'self.' : '';
+}

src/preamble.js

@@ -786,7 +786,7 @@ function instantiateSync(file, info) {
 }
 #endif
 
-#if PTHREADS && (LOAD_SOURCE_MAP || USE_OFFSET_CONVERTER)
+#if (PTHREADS || WASM_WORKERS) && (LOAD_SOURCE_MAP || USE_OFFSET_CONVERTER)
 // When using postMessage to send an object, it is processed by the structured
 // clone algorithm.  The prototype, and hence methods, on that object is then
 // lost. This function adds back the lost prototype.  This does not work with
@@ -1083,22 +1083,18 @@ function createWasm() {
   // path.
   if (Module['instantiateWasm']) {
 
-#if USE_OFFSET_CONVERTER && PTHREADS
-    if (ENVIRONMENT_IS_PTHREAD) {
+#if USE_OFFSET_CONVERTER
 #if ASSERTIONS
-      assert(Module['wasmOffsetData'], 'wasmOffsetData not found on Module object');
+{{{ runIfWorkerThread("assert(Module['wasmOffsetData'], 'wasmOffsetData not found on Module object');") }}}
 #endif
-      wasmOffsetConverter = resetPrototype(WasmOffsetConverter, Module['wasmOffsetData']);
-    }
+{{{ runIfWorkerThread("wasmOffsetConverter = resetPrototype(WasmOffsetConverter, Module['wasmOffsetData']);") }}}
 #endif
 
-#if LOAD_SOURCE_MAP && PTHREADS
-    if (ENVIRONMENT_IS_PTHREAD) {
+#if LOAD_SOURCE_MAP
 #if ASSERTIONS
-      assert(Module['wasmSourceMapData'], 'wasmSourceMapData not found on Module object');
+{{{ runIfWorkerThread("assert(Module['wasmSourceMapData'], 'wasmSourceMapData not found on Module object');") }}}
 #endif
-      wasmSourceMap = resetPrototype(WasmSourceMap, Module['wasmSourceMapData']);
-    }
+{{{ runIfWorkerThread("wasmSourceMap = resetPrototype(WasmSourceMap, Module['wasmSourceMapData']);") }}}
 #endif
 
     try {

src/wasm_worker.js

@@ -1,18 +1,51 @@
 // N.B. The contents of this file are duplicated in src/library_wasm_worker.js
 // in variable "_wasmWorkerBlobUrl" (where the contents are pre-minified) If
 // doing any changes to this file, be sure to update the contents there too.
-onmessage = function(d) {
+
+'use strict';
+
+#if ENVIRONMENT_MAY_BE_NODE
+// Node.js support
+var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions == 'object' && typeof process.versions.node == 'string';
+if (ENVIRONMENT_IS_NODE) {
+  // Create as web-worker-like an environment as we can.
+
+  var nodeWorkerThreads = require('worker_threads');
+
+  var parentPort = nodeWorkerThreads.parentPort;
+
+  parentPort.on('message', (data) => typeof onmessage === "function" && onmessage({ data: data }));
+
+  var fs = require('fs');
+
+  Object.assign(global, {
+    self: global,
+    require,
+    location: {
+      href: __filename
+    },
+    Worker: nodeWorkerThreads.Worker,
+    importScripts: (f) => (0, eval)(fs.readFileSync(f, 'utf8') + '//# sourceURL=' + f),
+    postMessage: (msg) => parentPort.postMessage(msg),
+    performance: global.performance || { now: Date.now },
+    addEventListener: (name, handler) => parentPort.on(name, handler),
+    removeEventListener: (name, handler) => parentPort.off(name, handler),
+  });
+}
+#endif // ENVIRONMENT_MAY_BE_NODE
+
+{{{ implicitSelf() }}}onmessage = function(d) {
 	// The first message sent to the Worker is always the bootstrap message.
 	// Drop this message listener, it served its purpose of bootstrapping
 	// the Wasm Module load, and is no longer needed. Let user code register
 	// any desired message handlers from now on.
-	onmessage = null;
+	{{{ implicitSelf() }}}onmessage = null;
 	d = d.data;
 #if !MODULARIZE
 	self.{{{ EXPORT_NAME }}} = d;
 #endif
 #if !MINIMAL_RUNTIME
-	d['instantiateWasm'] = (info, receiveInstance) => { var instance = new WebAssembly.Instance(d['wasm'], info); receiveInstance(instance, d['wasm']); return instance.exports; }
+	d['instantiateWasm'] = (info, receiveInstance) => { var instance = new WebAssembly.Instance(d['wasm'], info); return receiveInstance(instance, d['wasm']); }
 #endif
 	importScripts(d.js);
 #if MODULARIZE

test/test_core.py

@@ -9838,16 +9838,23 @@ def test_emscripten_async_load_script(self):
     self.run_process([FILE_PACKAGER, 'test.data', '--preload', 'file1.txt', 'file2.txt', '--from-emcc', '--js-output=script2.js'])
     self.do_runf(test_file('test_emscripten_async_load_script.c'), emcc_args=['-sFORCE_FILESYSTEM'])
 
+  def prep_wasm_worker_in_node(self):
+    # Auto exit after 3 seconds in Nodejs environment to get WASM Worker stdout
+    self.add_pre_run("setTimeout(()=>process.exit(), 3000);")
+
   @node_pthreads
   def test_wasm_worker_hello(self):
-    self.do_runf(test_file('wasm_worker/hello_wasm_worker.c'), emcc_args=['-sWASM_WORKERS'])
+    self.prep_wasm_worker_in_node()
+    self.do_run_in_out_file_test(test_file('wasm_worker/hello_wasm_worker.c'), emcc_args=['-sWASM_WORKERS'])
 
   @node_pthreads
   def test_wasm_worker_malloc(self):
-    self.do_runf(test_file('wasm_worker/malloc_wasm_worker.c'), emcc_args=['-sWASM_WORKERS'])
+    self.prep_wasm_worker_in_node()
+    self.do_run_in_out_file_test(test_file('wasm_worker/malloc_wasm_worker.c'), emcc_args=['-sWASM_WORKERS'])
 
   @node_pthreads
   def test_wasm_worker_wait_async(self):
+    self.prep_wasm_worker_in_node()
     self.do_runf(test_file('wasm_worker/wait_async.c'), emcc_args=['-sWASM_WORKERS'])
 
 

test/wasm_worker/hello_wasm_worker.out

@@ -0,0 +1 @@
+Hello from wasm worker!

test/wasm_worker/malloc_wasm_worker.out

@@ -0,0 +1 @@
+Hello from wasm worker!