diff --git a/embuilder.py b/embuilder.py index c6b01b677ee42..bc6b0d4828858 100755 --- a/embuilder.py +++ b/embuilder.py @@ -28,6 +28,7 @@ # Minimal subset of targets used by CI systems to build enough to useful MINIMAL_TASKS = [ + 'libbulkmemory', 'libcompiler_rt', 'libcompiler_rt-wasm-sjlj', 'libc', diff --git a/emcc.py b/emcc.py index ccfb59bc4f3a2..cbfeba26cd522 100755 --- a/emcc.py +++ b/emcc.py @@ -1599,6 +1599,9 @@ def phase_setup(options, state, newargs): if '-mbulk-memory' not in newargs: newargs += ['-mbulk-memory'] + if settings.SHARED_MEMORY: + settings.BULK_MEMORY = 1 + if 'DISABLE_EXCEPTION_CATCHING' in user_settings and 'EXCEPTION_CATCHING_ALLOWED' in user_settings: # If we get here then the user specified both DISABLE_EXCEPTION_CATCHING and EXCEPTION_CATCHING_ALLOWED # on the command line. This is no longer valid so report either an error or a warning (for @@ -2434,6 +2437,8 @@ def phase_linker_setup(options, state, newargs): settings.JS_LIBRARIES.append((0, shared.path_from_root('src', 'library_wasm_worker.js'))) settings.SUPPORTS_GLOBALTHIS = feature_matrix.caniuse(feature_matrix.Feature.GLOBALTHIS) + if not settings.BULK_MEMORY: + settings.BULK_MEMORY = feature_matrix.caniuse(feature_matrix.Feature.BULK_MEMORY) if settings.AUDIO_WORKLET: if not settings.SUPPORTS_GLOBALTHIS: @@ -3565,6 +3570,10 @@ def consume_arg_file(): settings.DISABLE_EXCEPTION_CATCHING = 1 settings.DISABLE_EXCEPTION_THROWING = 1 settings.WASM_EXCEPTIONS = 0 + elif arg == '-mbulk-memory': + settings.BULK_MEMORY = 1 + elif arg == '-mno-bulk-memory': + settings.BULK_MEMORY = 0 elif arg == '-fexceptions': # TODO Currently -fexceptions only means Emscripten EH. Switch to wasm # exception handling by default when -fexceptions is given when wasm diff --git a/src/library.js b/src/library.js index f38c244d972f2..f2afff44362cd 100644 --- a/src/library.js +++ b/src/library.js @@ -389,9 +389,11 @@ mergeInto(LibraryManager.library, { // variant, so we should never emit emscripten_memcpy_big() in the build. // In STANDALONE_WASM we avoid the emscripten_memcpy_big dependency so keep // the wasm file standalone. + // In BULK_MEMORY mode we include native versions of these functions based + // on memory.fill and memory.copy. // In MAIN_MODULE=1 or EMCC_FORCE_STDLIBS mode all of libc is force included // so we cannot override parts of it, and therefore cannot use libc_optz. -#if (SHRINK_LEVEL < 2 || LINKABLE || process.env.EMCC_FORCE_STDLIBS) && !STANDALONE_WASM +#if (SHRINK_LEVEL < 2 || LINKABLE || process.env.EMCC_FORCE_STDLIBS) && !STANDALONE_WASM && !BULK_MEMORY #if MIN_CHROME_VERSION < 45 || MIN_EDGE_VERSION < 14 || MIN_FIREFOX_VERSION < 34 || MIN_IE_VERSION != TARGET_NOT_SUPPORTED || MIN_SAFARI_VERSION < 100101 // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray/copyWithin lists browsers that support TypedArray.prototype.copyWithin, but it diff --git a/src/settings_internal.js b/src/settings_internal.js index 040cb40d8e7c1..cc4f7506ecd90 100644 --- a/src/settings_internal.js +++ b/src/settings_internal.js @@ -256,3 +256,5 @@ var POST_JS_FILES = []; // Set when -pthread / -sPTHREADS is passed var PTHREADS = false; + +var BULK_MEMORY = false; diff --git a/system/lib/libc/emscripten_internal.h b/system/lib/libc/emscripten_internal.h index 92dbe29e43e75..4aff06e6b1a35 100644 --- a/system/lib/libc/emscripten_internal.h +++ b/system/lib/libc/emscripten_internal.h @@ -30,6 +30,7 @@ extern "C" { void emscripten_memcpy_big(void* __restrict__ dest, const void* __restrict__ src, size_t n) EM_IMPORT(emscripten_memcpy_big); +void emscripten_memset_big(void* ptr, char value, size_t n); void emscripten_notify_memory_growth(size_t memory_index); diff --git a/system/lib/libc/emscripten_memcpy.c b/system/lib/libc/emscripten_memcpy.c index 3a3f091a5def6..d226b3eebad51 100644 --- a/system/lib/libc/emscripten_memcpy.c +++ b/system/lib/libc/emscripten_memcpy.c @@ -29,7 +29,7 @@ static void *__memcpy(void *restrict dest, const void *restrict src, size_t n) { unsigned char *block_aligned_d_end; unsigned char *d_end; -#ifndef EMSCRIPTEN_STANDALONE_WASM +#if !defined(EMSCRIPTEN_STANDALONE_WASM) || defined(__wasm_bulk_memory__) if (n >= 512) { emscripten_memcpy_big(dest, src, n); return dest; diff --git a/system/lib/libc/emscripten_memcpy_big.S b/system/lib/libc/emscripten_memcpy_big.S new file mode 100644 index 0000000000000..2c3bdfbe674d6 --- /dev/null +++ b/system/lib/libc/emscripten_memcpy_big.S @@ -0,0 +1,14 @@ +#ifdef __wasm64__ +#define PTR i64 +#else +#define PTR i32 +#endif + +.globl emscripten_memcpy_big +emscripten_memcpy_big: + .functype emscripten_memcpy_big (PTR, PTR, PTR) -> () + local.get 0 + local.get 1 + local.get 2 + memory.copy 0, 0 + end_function diff --git a/system/lib/libc/emscripten_memset.c b/system/lib/libc/emscripten_memset.c index 048390e4fc3cb..6109d52eddf35 100644 --- a/system/lib/libc/emscripten_memset.c +++ b/system/lib/libc/emscripten_memset.c @@ -1,23 +1,44 @@ -// XXX EMSCRIPTEN ASAN: build an uninstrumented version of memset -#if defined(__EMSCRIPTEN__) && defined(__has_feature) -#if __has_feature(address_sanitizer) -#define memset __attribute__((no_sanitize("address"))) emscripten_builtin_memset -#endif +#include "emscripten_internal.h" // for emscripten_memset_big + +#if defined(__has_feature) && __has_feature(address_sanitizer) +// build an uninstrumented version of memset +__attribute__((no_sanitize("address"))) void *__musl_memset(void *str, int c, size_t n); +__attribute__((no_sanitize("address"))) void *__memset(void *str, int c, size_t n); #endif -#ifdef EMSCRIPTEN_OPTIMIZE_FOR_OZ +__attribute__((__weak__)) void *__musl_memset(void *str, int c, size_t n); +__attribute__((__weak__)) void *__memset(void *str, int c, size_t n); -#include +#ifdef EMSCRIPTEN_OPTIMIZE_FOR_OZ -void *memset(void *str, int c, size_t n) { +void *__memset(void *str, int c, size_t n) { unsigned char *s = (unsigned char *)str; #pragma clang loop unroll(disable) while(n--) *s++ = c; return str; } +#elif defined(__wasm_bulk_memory__) + +#define memset __musl_memset +#include "musl/src/string/memset.c" +#undef memset + +void *__memset(void *str, int c, size_t n) { + if (n >= 512) { + emscripten_memset_big(str, c, n); + return str; + } + return __musl_memset(str, c, n); +} + #else +#define memset __memset #include "musl/src/string/memset.c" +#undef memset #endif + +weak_alias(__memset, emscripten_builtin_memset); +weak_alias(__memset, memset); diff --git a/system/lib/libc/emscripten_memset_big.S b/system/lib/libc/emscripten_memset_big.S new file mode 100644 index 0000000000000..ed765c3190d2c --- /dev/null +++ b/system/lib/libc/emscripten_memset_big.S @@ -0,0 +1,14 @@ +#ifdef __wasm64__ +#define PTR i64 +#else +#define PTR i32 +#endif + +.globl emscripten_memset_big +emscripten_memset_big: + .functype emscripten_memset_big (PTR, i32, PTR) -> () + local.get 0 + local.get 1 + local.get 2 + memory.fill 0 + end_function diff --git a/system/lib/standalone/standalone.c b/system/lib/standalone/standalone.c index 200002dce1e1b..7f2185e8da8da 100644 --- a/system/lib/standalone/standalone.c +++ b/system/lib/standalone/standalone.c @@ -152,7 +152,7 @@ int emscripten_resize_heap(size_t size) { } double emscripten_get_now(void) { - return (1000 * clock()) / (double)CLOCKS_PER_SEC; + return (1000ll * clock()) / (double)CLOCKS_PER_SEC; } // C++ ABI diff --git a/test/other/metadce/test_metadce_hello_O0.funcs b/test/other/metadce/test_metadce_hello_O0.funcs index b12d0bc6b6c73..684921992a5f7 100644 --- a/test/other/metadce/test_metadce_hello_O0.funcs +++ b/test/other/metadce/test_metadce_hello_O0.funcs @@ -9,6 +9,7 @@ $__lock $__lockfile $__lshrti3 $__memcpy +$__memset $__ofl_lock $__ofl_unlock $__original_main @@ -41,7 +42,6 @@ $isdigit $legalstub$dynCall_jiji $main $memchr -$memset $out $pad $pop_arg diff --git a/test/other/metadce/test_metadce_minimal_pthreads.funcs b/test/other/metadce/test_metadce_minimal_pthreads.funcs index 86cf4f2716c41..b73e40caf9df9 100644 --- a/test/other/metadce/test_metadce_minimal_pthreads.funcs +++ b/test/other/metadce/test_metadce_minimal_pthreads.funcs @@ -1,6 +1,7 @@ $__emscripten_stdout_seek $__errno_location $__memcpy +$__memset $__pthread_mutex_lock $__pthread_mutex_trylock $__pthread_mutex_unlock @@ -63,7 +64,6 @@ $get_tasks_for_thread $init_file_lock $init_mparams $main -$memset $nodtor $pthread_attr_destroy $receive_notification diff --git a/test/other/test_memops_bulk_memory.c b/test/other/test_memops_bulk_memory.c new file mode 100644 index 0000000000000..5f6dc0b6874c3 --- /dev/null +++ b/test/other/test_memops_bulk_memory.c @@ -0,0 +1,13 @@ +#include +#include + +const char *hello = "hello"; +const char *world = "world"; + +int main() { + char buffer[100]; + memset(buffer, 'a', 100); + memcpy(buffer, hello, strlen(hello) + 1); + assert(strcmp(buffer, hello) == 0); + return 0; +} diff --git a/test/test_other.py b/test/test_other.py index 9ca7e4efe0289..7d70fb9c13768 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -12219,7 +12219,7 @@ def test_standard_library_mapping(self): # Test the `-l` flags on the command line get mapped the correct libraries variant self.run_process([EMBUILDER, 'build', 'libc-mt-debug', 'libcompiler_rt-mt', 'libdlmalloc-mt']) - libs = ['-lc', '-lcompiler_rt', '-lmalloc'] + libs = ['-lc', '-lbulkmemory', '-lcompiler_rt', '-lmalloc'] err = self.run_process([EMCC, test_file('hello_world.c'), '-pthread', '-nodefaultlibs', '-v'] + libs, stderr=PIPE).stderr # Check that the linker was run with `-mt` variants because `-pthread` was passed. @@ -13412,3 +13412,30 @@ def test_wasi_random_get(self): @requires_node def test_wasi_sched_yield(self): self.run_wasi_test_suite_test('wasi_sched_yield') + + def test_memops_bulk_memory(self): + self.emcc_args += ['--profiling-funcs', '-fno-builtin'] + + def run(args, expect_bulk_mem): + self.do_runf(test_file('other/test_memops_bulk_memory.c'), emcc_args=args) + funcs = self.parse_wasm('test_memops_bulk_memory.wasm')[2] + js = read_file('test_memops_bulk_memory.js') + if expect_bulk_mem: + self.assertNotContained('_emscripten_memcpy_big', js) + self.assertIn('$emscripten_memcpy_big', funcs) + else: + self.assertContained('_emscripten_memcpy_big', js) + self.assertNotIn('$emscripten_memcpy_big', funcs) + + # By default we expect to find _emscripten_memcpy_big in the generaed JS and not in the + # native code. + run([], expect_bulk_mem=False) + + # With bulk memory enabled we expect *not* to find it. + run(['-mbulk-memory'], expect_bulk_mem=True) + + run(['-mbulk-memory', '-mno-bulk-memory'], expect_bulk_mem=False) + + # -pthread implicitly enables bulk memory too. + self.setup_node_pthreads() + run(['-pthread'], expect_bulk_mem=True) diff --git a/tools/settings.py b/tools/settings.py index 429020bf12dbd..10d6ca06ebff6 100644 --- a/tools/settings.py +++ b/tools/settings.py @@ -71,6 +71,7 @@ 'DEFAULT_TO_CXX', 'WASM_OBJECT_FILES', 'WASM_WORKERS', + 'BULK_MEMORY', # Internal settings used during compilation 'EXCEPTION_CATCHING_ALLOWED', diff --git a/tools/system_libs.py b/tools/system_libs.py index 7bebfb22b6bbc..b0cca903a4963 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1274,6 +1274,17 @@ def can_use(self): not settings.LINKABLE and not os.environ.get('EMCC_FORCE_STDLIBS') +class libbulkmemory(MuslInternalLibrary, AsanInstrumentedLibrary): + name = 'libbulkmemory' + src_dir = 'system/lib/libc' + src_files = ['emscripten_memcpy.c', 'emscripten_memset.c', + 'emscripten_memcpy_big.S', 'emscripten_memset_big.S'] + cflags = ['-mbulk-memory'] + + def can_use(self): + return super(libbulkmemory, self).can_use() and settings.BULK_MEMORY + + class libprintf_long_double(libc): name = 'libprintf_long_double' cflags = ['-DEMSCRIPTEN_PRINTF_LONG_DOUBLE'] @@ -1945,7 +1956,7 @@ def get_files(self): '__main_void.c']) files += files_in_path( path='system/lib/libc', - filenames=['emscripten_memcpy.c']) + filenames=['emscripten_memcpy.c', 'emscripten_memset.c']) # It is more efficient to use JS methods for time, normally. files += files_in_path( path='system/lib/libc/musl/src/time', @@ -2154,7 +2165,8 @@ def add_sanitizer_libs(): if settings.SHRINK_LEVEL >= 2 and not settings.LINKABLE and \ not os.environ.get('EMCC_FORCE_STDLIBS'): add_library('libc_optz') - + if settings.BULK_MEMORY: + add_library('libbulkmemory') if settings.STANDALONE_WASM: add_library('libstandalonewasm') if settings.ALLOW_UNIMPLEMENTED_SYSCALLS: