From 092d8b0f26e6e39ae347003a0cc1f61348ff71b5 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Thu, 30 May 2019 10:39:31 -0700 Subject: [PATCH 01/62] Use emscripten_builtin_malloc/free in select library code --- emcc.py | 2 +- src/deps_info.json | 8 ++++---- src/library.js | 6 +++--- src/library_syscall.js | 7 ++++--- src/runtime_strings.js | 4 ++-- system/lib/dlmalloc.c | 19 +++++++++++++++++-- system/lib/emmalloc.cpp | 1 + tools/system_libs.py | 2 +- 8 files changed, 33 insertions(+), 16 deletions(-) diff --git a/emcc.py b/emcc.py index 6cec33e0562ac..bd043b399087f 100755 --- a/emcc.py +++ b/emcc.py @@ -1199,7 +1199,7 @@ def check(input_file): if not shared.Settings.ONLY_MY_CODE and not shared.Settings.MINIMAL_RUNTIME: # Always need malloc and free to be kept alive and exported, for internal use and other modules - shared.Settings.EXPORTED_FUNCTIONS += ['_malloc', '_free'] + shared.Settings.EXPORTED_FUNCTIONS += ['_malloc', '_free', '_emscripten_builtin_malloc', '_emscripten_builtin_free'] if shared.Settings.WASM_BACKEND: # setjmp/longjmp and exception handling JS code depends on this so we # include it by default. Should be eliminated by meta-DCE if unused. diff --git a/src/deps_info.json b/src/deps_info.json index 4187f5d1c3e8f..49447e06b6498 100644 --- a/src/deps_info.json +++ b/src/deps_info.json @@ -1,8 +1,8 @@ { "arc4random": ["rand"], "freopen": ["free"], - "munmap": ["free"], - "getenv": ["malloc", "free", "_get_environ"], + "munmap": ["emscripten_builtin_free"], + "getenv": ["emscripten_builtin_malloc", "emscripten_builtin_free", "_get_environ"], "clearenv": ["_get_environ"], "setenv": ["_get_environ"], "unsetenv": ["_get_environ"], @@ -10,7 +10,7 @@ "environ": ["_get_environ"], "fmtmsg": ["_get_environ"], "get_current_dir_name": ["_get_environ"], - "SDL_getenv": ["malloc", "free", "_get_environ"], + "SDL_getenv": ["emscripten_builtin_malloc", "emscripten_builtin_free", "_get_environ"], "SDL_putenv": ["_get_environ"], "dlerror": ["malloc", "free"], "readdir": ["malloc"], @@ -19,7 +19,7 @@ "realloc": ["malloc", "free"], "getlogin": ["malloc", "_get_environ"], "tmpnam": ["malloc"], - "mmap": ["memalign"], + "mmap": ["emscripten_builtin_memalign"], "realpath": ["malloc"], "strerror": ["malloc"], "__ctype_b_loc": ["malloc"], diff --git a/src/library.js b/src/library.js index 7e06ba48219f2..a1ef61eac788c 100644 --- a/src/library.js +++ b/src/library.js @@ -921,7 +921,7 @@ LibraryManager.library = { {{{ makeSetValue('envPtr', 'strings.length * ptrSize', '0', 'i8*') }}}; }, $ENV: {}, - getenv__deps: ['$ENV'], + getenv__deps: ['$ENV', 'emscripten_builtin_malloc', 'emscripten_builtin_free'], getenv__proxy: 'sync', getenv__sig: 'ii', getenv: function(name) { @@ -931,8 +931,8 @@ LibraryManager.library = { name = UTF8ToString(name); if (!ENV.hasOwnProperty(name)) return 0; - if (_getenv.ret) _free(_getenv.ret); - _getenv.ret = allocateUTF8(ENV[name]); + if (_getenv.ret) _emscripten_builtin_free(_getenv.ret); + _getenv.ret = allocateUTF8(ENV[name], _emscripten_builtin_malloc); return _getenv.ret; }, // Alias for sanitizers which intercept getenv. diff --git a/src/library_syscall.js b/src/library_syscall.js index f717ffdfcc3b1..aea529d981f4d 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -444,6 +444,7 @@ var SyscallsLibrary = { var path = SYSCALLS.getStr(), buf = SYSCALLS.get(), bufsize = SYSCALLS.get(); return SYSCALLS.doReadlink(path, buf, bufsize); }, + __syscall91__deps: ['emscripten_builtin_free'], __syscall91: function(which, varargs) { // munmap var addr = SYSCALLS.get(), len = SYSCALLS.get(); // TODO: support unmmap'ing parts of allocations @@ -455,7 +456,7 @@ var SyscallsLibrary = { FS.munmap(stream); SYSCALLS.mappings[addr] = null; if (info.allocated) { - _free(info.malloc); + _emscripten_builtin_free(info.malloc); } } return 0; @@ -975,14 +976,14 @@ var SyscallsLibrary = { {{{ makeSetValue('rlim', C_STRUCTS.rlimit.rlim_max + 4, '-1', 'i32') }}}; // RLIM_INFINITY return 0; // just report no limits }, - __syscall192__deps: ['memalign'], + __syscall192__deps: ['emscripten_builtin_memalign'], __syscall192: function(which, varargs) { // mmap2 var addr = SYSCALLS.get(), len = SYSCALLS.get(), prot = SYSCALLS.get(), flags = SYSCALLS.get(), fd = SYSCALLS.get(), off = SYSCALLS.get() off <<= 12; // undo pgoffset var ptr; var allocated = false; if (fd === -1) { - ptr = _memalign(PAGE_SIZE, len); + ptr = _emscripten_builtin_memalign(PAGE_SIZE, len); if (!ptr) return -{{{ cDefine('ENOMEM') }}}; _memset(ptr, 0, len); allocated = true; diff --git a/src/runtime_strings.js b/src/runtime_strings.js index 6c0b61374e21f..bbb7f147f929f 100644 --- a/src/runtime_strings.js +++ b/src/runtime_strings.js @@ -376,9 +376,9 @@ function lengthBytesUTF32(str) { // Allocate heap space for a JS string, and write it there. // It is the responsibility of the caller to free() that memory. -function allocateUTF8(str) { +function allocateUTF8(str, allocator) { var size = lengthBytesUTF8(str) + 1; - var ret = _malloc(size); + var ret = (allocator || _malloc)(size); if (ret) stringToUTF8Array(str, HEAP8, ret, size); return ret; } diff --git a/system/lib/dlmalloc.c b/system/lib/dlmalloc.c index acdd77c97fcea..02f45acad04b1 100644 --- a/system/lib/dlmalloc.c +++ b/system/lib/dlmalloc.c @@ -850,6 +850,19 @@ extern "C" { /* ------------------- Declarations of public routines ------------------- */ #ifndef USE_DL_PREFIX +// XXX Emscripten XXX + +#ifdef __EMSCRIPTEN__ +void* malloc(size_t) __attribute__((weak, alias("dlmalloc"))); +void free(void*) __attribute__((weak, alias("dlfree"))); +void* calloc(size_t, size_t) __attribute__((weak, alias("dlcalloc"))); +void* realloc(void*, size_t) __attribute__((weak, alias("dlrealloc"))); +void* realloc_in_place(void*, size_t) __attribute__((weak, alias("dlrealloc_in_place"))); +void* memalign(size_t, size_t) __attribute__((weak, alias("dlmemalign"))); +int posix_memalign(void**, size_t, size_t) __attribute__((weak, alias("dlposix_memalign"))); +void* valloc(size_t) __attribute__((weak, alias("dlvalloc"))); +void* pvalloc(size_t) __attribute__((weak, alias("dlpvalloc"))); +#else #define dlcalloc calloc #define dlfree free #define dlmalloc malloc @@ -859,6 +872,7 @@ extern "C" { #define dlrealloc_in_place realloc_in_place #define dlvalloc valloc #define dlpvalloc pvalloc +#endif #define dlmallinfo mallinfo #define dlmallopt mallopt #define dlmalloc_trim malloc_trim @@ -6032,8 +6046,9 @@ int mspace_mallopt(int param_number, int value) { // and dlfree from this file. // This allows an easy mechanism for hooking into memory allocation. #if defined(__EMSCRIPTEN__) && !ONLY_MSPACES -extern __typeof(malloc) emscripten_builtin_malloc __attribute__((weak, alias("malloc"))); -extern __typeof(free) emscripten_builtin_free __attribute__((weak, alias("free"))); +extern __typeof(malloc) emscripten_builtin_malloc __attribute__((weak, alias("dlmalloc"))); +extern __typeof(free) emscripten_builtin_free __attribute__((weak, alias("dlfree"))); +extern __typeof(memalign) emscripten_builtin_memalign __attribute__((weak, alias("dlmemalign"))); #endif /* -------------------- Alternative MORECORE functions ------------------- */ diff --git a/system/lib/emmalloc.cpp b/system/lib/emmalloc.cpp index 40fb34345ed12..b38e18b83954d 100644 --- a/system/lib/emmalloc.cpp +++ b/system/lib/emmalloc.cpp @@ -1201,6 +1201,7 @@ struct mallinfo mallinfo() { #if defined(__EMSCRIPTEN__) extern __typeof(malloc) emscripten_builtin_malloc __attribute__((weak, alias("malloc"))); extern __typeof(free) emscripten_builtin_free __attribute__((weak, alias("free"))); +extern __typeof(memalign) emscripten_builtin_memalign __attribute__((weak, alias("memalign"))); #endif } // extern "C" diff --git a/tools/system_libs.py b/tools/system_libs.py index d71aac6ffad88..ee815d4f80bbf 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -830,7 +830,7 @@ def do_create(): # TODO: handle this dependency better add_library(system_libs_map['libc++abi']) # FIXME: add this to deps_info.json and make add_back_deps work with libraries. - shared.Settings.EXPORTED_FUNCTIONS.append('_memalign') + shared.Settings.EXPORTED_FUNCTIONS += ['_emscripten_builtin_malloc', '_emscripten_builtin_free', '_emscripten_builtin_memalign'] libs_to_link.sort(key=lambda x: x[0].endswith('.a')) # make sure to put .a files at the end. From d417ff3363a9c9088a8fcf52c4537a9c3d1d0740 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Thu, 30 May 2019 14:11:08 -0700 Subject: [PATCH 02/62] Make emscripten_builtin_* strong since we need them to be the original --- system/lib/dlmalloc.c | 6 +++--- system/lib/emmalloc.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/system/lib/dlmalloc.c b/system/lib/dlmalloc.c index 02f45acad04b1..54ef05b2c2f8a 100644 --- a/system/lib/dlmalloc.c +++ b/system/lib/dlmalloc.c @@ -6046,9 +6046,9 @@ int mspace_mallopt(int param_number, int value) { // and dlfree from this file. // This allows an easy mechanism for hooking into memory allocation. #if defined(__EMSCRIPTEN__) && !ONLY_MSPACES -extern __typeof(malloc) emscripten_builtin_malloc __attribute__((weak, alias("dlmalloc"))); -extern __typeof(free) emscripten_builtin_free __attribute__((weak, alias("dlfree"))); -extern __typeof(memalign) emscripten_builtin_memalign __attribute__((weak, alias("dlmemalign"))); +extern __typeof(malloc) emscripten_builtin_malloc __attribute__((alias("dlmalloc"))); +extern __typeof(free) emscripten_builtin_free __attribute__((alias("dlfree"))); +extern __typeof(memalign) emscripten_builtin_memalign __attribute__((alias("dlmemalign"))); #endif /* -------------------- Alternative MORECORE functions ------------------- */ diff --git a/system/lib/emmalloc.cpp b/system/lib/emmalloc.cpp index b38e18b83954d..da35f2c88c076 100644 --- a/system/lib/emmalloc.cpp +++ b/system/lib/emmalloc.cpp @@ -1199,9 +1199,9 @@ struct mallinfo mallinfo() { // and free from this file. // This allows an easy mechanism for hooking into memory allocation. #if defined(__EMSCRIPTEN__) -extern __typeof(malloc) emscripten_builtin_malloc __attribute__((weak, alias("malloc"))); -extern __typeof(free) emscripten_builtin_free __attribute__((weak, alias("free"))); -extern __typeof(memalign) emscripten_builtin_memalign __attribute__((weak, alias("memalign"))); +extern __typeof(malloc) emscripten_builtin_malloc __attribute__((alias("malloc"))); +extern __typeof(free) emscripten_builtin_free __attribute__((alias("free"))); +extern __typeof(memalign) emscripten_builtin_memalign __attribute__((alias("memalign"))); #endif } // extern "C" From 789679dece97216d16c143545e714b96f55a60fc Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Thu, 30 May 2019 15:40:58 -0700 Subject: [PATCH 03/62] Fix and parameterize emmalloc test --- system/lib/emmalloc.cpp | 2 +- tests/test_core.py | 23 +++++++++-------------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/system/lib/emmalloc.cpp b/system/lib/emmalloc.cpp index da35f2c88c076..e7a0b4dda52c4 100644 --- a/system/lib/emmalloc.cpp +++ b/system/lib/emmalloc.cpp @@ -1198,7 +1198,7 @@ struct mallinfo mallinfo() { // in their code, and make those replacements refer to the original malloc // and free from this file. // This allows an easy mechanism for hooking into memory allocation. -#if defined(__EMSCRIPTEN__) +#if defined(__EMSCRIPTEN__) && !defined(TESTING_EMMALLOC) extern __typeof(malloc) emscripten_builtin_malloc __attribute__((alias("malloc"))); extern __typeof(free) emscripten_builtin_free __attribute__((alias("free"))); extern __typeof(memalign) emscripten_builtin_memalign __attribute__((alias("memalign"))); diff --git a/tests/test_core.py b/tests/test_core.py index 378ab2a9fa699..93f1efe807827 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -750,24 +750,19 @@ def test_structs(self): def test_mallocstruct(self): self.do_run(self.gen_struct_src.replace('{{gen_struct}}', '(S*)malloc(sizeof(S))').replace('{{del_struct}}', 'free'), '*51,62*') - def test_emmalloc(self): + @parameterized({ + 'normal': [], + 'debug': ['-DEMMALLOC_DEBUG'], + 'debug_log': ['-DEMMALLOC_DEBUG', '-DEMMALLOC_DEBUG_LOG', '-DRANDOM_ITERS=130'], + }) + def test_emmalloc(self, *args): # in newer clang+llvm, the internal calls to malloc in emmalloc may be optimized under # the assumption that they are external, so like in system_libs.py where we build # malloc, we need to disable builtin here too - self.emcc_args += ['-fno-builtin'] + self.emcc_args += ['-fno-builtin', '-DTESTING_EMMALLOC'] + list(args) - def test(): - self.do_run(open(path_from_root('system', 'lib', 'emmalloc.cpp')).read() + open(path_from_root('tests', 'core', 'test_emmalloc.cpp')).read(), - open(path_from_root('tests', 'core', 'test_emmalloc.txt')).read()) - print('normal') - test() - print('debug') - self.emcc_args += ['-DEMMALLOC_DEBUG'] - test() - print('debug log') - self.emcc_args += ['-DEMMALLOC_DEBUG_LOG'] - self.emcc_args += ['-DRANDOM_ITERS=130'] - test() + self.do_run(open(path_from_root('system', 'lib', 'emmalloc.cpp')).read() + open(path_from_root('tests', 'core', 'test_emmalloc.cpp')).read(), + open(path_from_root('tests', 'core', 'test_emmalloc.txt')).read()) def test_newstruct(self): self.do_run(self.gen_struct_src.replace('{{gen_struct}}', 'new S').replace('{{del_struct}}', 'delete'), '*51,62*') From 9e95a5668d95218aee57ea9abcbd7ca6ac64820c Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 29 May 2019 11:29:36 -0700 Subject: [PATCH 04/62] Make interception build on emscripten --- system/lib/compiler-rt/lib/interception/interception.h | 5 +++-- .../compiler-rt/lib/interception/interception_linux.cc | 7 ++++--- .../compiler-rt/lib/interception/interception_linux.h | 7 ++++--- tools/system_libs.py | 9 +++++++++ 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/system/lib/compiler-rt/lib/interception/interception.h b/system/lib/compiler-rt/lib/interception/interception.h index 87b2365fd7671..f80765a7ea654 100644 --- a/system/lib/compiler-rt/lib/interception/interception.h +++ b/system/lib/compiler-rt/lib/interception/interception.h @@ -19,7 +19,8 @@ #if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \ !SANITIZER_NETBSD && !SANITIZER_OPENBSD && !SANITIZER_WINDOWS && \ - !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_SOLARIS + !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_SOLARIS && \ + !SANITIZER_EMSCRIPTEN # error "Interception doesn't work on this operating system." #endif @@ -276,7 +277,7 @@ typedef unsigned long uptr; // NOLINT #define INCLUDED_FROM_INTERCEPTION_LIB #if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ - SANITIZER_OPENBSD || SANITIZER_SOLARIS + SANITIZER_OPENBSD || SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN # include "interception_linux.h" # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) diff --git a/system/lib/compiler-rt/lib/interception/interception_linux.cc b/system/lib/compiler-rt/lib/interception/interception_linux.cc index 26bfcd8f6794d..f0156c2c399c5 100644 --- a/system/lib/compiler-rt/lib/interception/interception_linux.cc +++ b/system/lib/compiler-rt/lib/interception/interception_linux.cc @@ -15,7 +15,7 @@ #include "interception.h" #if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ - SANITIZER_OPENBSD || SANITIZER_SOLARIS + SANITIZER_OPENBSD || SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN #include // for dlsym() and dlvsym() @@ -42,8 +42,9 @@ bool GetRealFunctionAddress(const char *func_name, uptr *func_addr, return real == wrapper; } -// Android and Solaris do not have dlvsym -#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD +// Android, Solaris, OpenBSD and emscripten do not have dlvsym +#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD && \ + !SANITIZER_EMSCRIPTEN void *GetFuncAddrVer(const char *func_name, const char *ver) { return dlvsym(RTLD_NEXT, func_name, ver); } diff --git a/system/lib/compiler-rt/lib/interception/interception_linux.h b/system/lib/compiler-rt/lib/interception/interception_linux.h index 765a186e58232..b5199ee32771e 100644 --- a/system/lib/compiler-rt/lib/interception/interception_linux.h +++ b/system/lib/compiler-rt/lib/interception/interception_linux.h @@ -13,7 +13,7 @@ //===----------------------------------------------------------------------===// #if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ - SANITIZER_OPENBSD || SANITIZER_SOLARIS + SANITIZER_OPENBSD || SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN #if !defined(INCLUDED_FROM_INTERCEPTION_LIB) # error "interception_linux.h should be included from interception library only" @@ -35,8 +35,9 @@ void *GetFuncAddrVer(const char *func_name, const char *ver); (::__interception::uptr) & (func), \ (::__interception::uptr) & WRAP(func)) -// Android, Solaris and OpenBSD do not have dlvsym -#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD +// Android, Solaris, OpenBSD and emscripten do not have dlvsym +#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD && \ + !SANITIZER_EMSCRIPTEN #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ (::__interception::real_##func = (func##_type)( \ unsigned long)::__interception::GetFuncAddrVer(#func, symver)) diff --git a/tools/system_libs.py b/tools/system_libs.py index ee815d4f80bbf..7d9f86d66d21c 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -603,6 +603,15 @@ def create_wasm_ubsan_rt(libname): '-DUBSAN_CAN_USE_CXXABI' ]) + def create_wasm_interception_rt(libname): + files = glob_in_path( + path_components=['system', 'lib', 'compiler-rt', 'lib', 'interception'], + glob_pattern='*.cc', + ) + return create_wasm_rt_lib(libname, files, extra_flags=[ + '-I', shared.path_from_root('system', 'lib', 'compiler-rt', 'lib'), '-std=c++11' + ]) + def create_wasm_libc_rt(libname): return create_wasm_rt_lib(libname, get_wasm_libc_rt_files()) From e4e778be481246045b7b1097800261c6ae01990e Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 29 May 2019 12:58:54 -0700 Subject: [PATCH 05/62] Implement LSan linking --- emcc.py | 3 +++ src/settings.js | 4 ++++ tools/system_libs.py | 24 ++++++++++++++++++++---- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/emcc.py b/emcc.py index bd043b399087f..9a22a6acb8d25 100755 --- a/emcc.py +++ b/emcc.py @@ -1482,6 +1482,9 @@ def check(input_file): else: shared.Settings.UBSAN_RUNTIME = 2 + if 'leak' in sanitize: + shared.Settings.USE_LSAN = True + if sanitize and '-g4' in args: shared.Settings.LOAD_SOURCE_MAP = 1 diff --git a/src/settings.js b/src/settings.js index 7daee70b07b6b..a64012f7fe411 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1431,6 +1431,10 @@ var WASM2JS = 0; // -fsanitize=undefined. To use minimal runtime, also pass `-fsanitize-minimal-runtime`. var UBSAN_RUNTIME = 0; +// Whether we should link in LSan's runtime library. This is intended to be used invoked` +// by -fsanitize=leak instead of used directly. +var USE_LSAN = 0; + // Whether we should load the WASM source map at runtime. // This is enabled automatically when using -g4 with sanitizers. var LOAD_SOURCE_MAP = 0; diff --git a/tools/system_libs.py b/tools/system_libs.py index 7d9f86d66d21c..a25ba334ed5c0 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -612,6 +612,15 @@ def create_wasm_interception_rt(libname): '-I', shared.path_from_root('system', 'lib', 'compiler-rt', 'lib'), '-std=c++11' ]) + def create_wasm_lsan_rt(libname): + files = glob_in_path( + path_components=['system', 'lib', 'compiler-rt', 'lib', 'lsan'], + glob_pattern='*.cc', + ) + return create_wasm_rt_lib(libname, files, extra_flags=[ + '-I', shared.path_from_root('system', 'lib', 'compiler-rt', 'lib'), '-std=c++11' + ]) + def create_wasm_libc_rt(libname): return create_wasm_rt_lib(libname, get_wasm_libc_rt_files()) @@ -831,15 +840,22 @@ def do_create(): libs_to_link.append((shared.Cache.get('libcompiler_rt_wasm.a', lambda: create_wasm_compiler_rt('libcompiler_rt_wasm.a')), False)) libs_to_link.append((shared.Cache.get('libc_rt_wasm.a', lambda: create_wasm_libc_rt('libc_rt_wasm.a')), False)) + need_common_san = False + if shared.Settings.UBSAN_RUNTIME == 1: libs_to_link.append((shared.Cache.get('libubsan_minimal_rt_wasm.a', lambda: create_wasm_ubsan_minimal_rt('libubsan_minimal_rt_wasm.a')), False)) elif shared.Settings.UBSAN_RUNTIME == 2: - libs_to_link.append((shared.Cache.get('libsanitizer_common_rt_wasm.a', lambda: create_wasm_common_san_rt('libsanitizer_common_rt_wasm.a')), False)) + need_common_san = True libs_to_link.append((shared.Cache.get('libubsan_rt_wasm.a', lambda: create_wasm_ubsan_rt('libubsan_rt_wasm.a')), False)) - # TODO: handle this dependency better + + if shared.Settings.USE_LSAN: + need_common_san = True + libs_to_link.append((shared.Cache.get('liblsan_rt_wasm.a', lambda: create_wasm_lsan_rt('liblsan_rt_wasm.a')), True)) + + if need_common_san: add_library(system_libs_map['libc++abi']) - # FIXME: add this to deps_info.json and make add_back_deps work with libraries. - shared.Settings.EXPORTED_FUNCTIONS += ['_emscripten_builtin_malloc', '_emscripten_builtin_free', '_emscripten_builtin_memalign'] + libs_to_link.append((shared.Cache.get('libsanitizer_common_rt_wasm.a', lambda: create_wasm_common_san_rt('libsanitizer_common_rt_wasm.a')), False)) + shared.Settings.EXPORTED_FUNCTIONS += ['_emscripten_builtin_memalign'] libs_to_link.sort(key=lambda x: x[0].endswith('.a')) # make sure to put .a files at the end. From 79fba27b19558730ec0cd3577c431244c8d43012 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 29 May 2019 11:41:04 -0700 Subject: [PATCH 06/62] Some LSan work --- emcc.py | 3 +- src/library_pthread_stub.js | 4 + .../lib/interception/interception.h | 2 +- .../compiler-rt/lib/lsan/lsan_allocator.cc | 2 +- .../lib/compiler-rt/lib/lsan/lsan_allocator.h | 2 +- .../lib/compiler-rt/lib/lsan/lsan_common.cc | 12 +- system/lib/compiler-rt/lib/lsan/lsan_common.h | 2 + .../compiler-rt/lib/lsan/lsan_emscripten.cc | 103 ++++++++++++++++++ .../compiler-rt/lib/lsan/lsan_interceptors.cc | 2 + system/lib/compiler-rt/lib/lsan/lsan_linux.cc | 2 +- .../sanitizer_common/sanitizer_emscripten.cc | 16 +++ .../lib/sanitizer_common/sanitizer_linux.cc | 2 + .../sanitizer_posix_libcdep.cc | 2 + 13 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc diff --git a/emcc.py b/emcc.py index 9a22a6acb8d25..b3d47884737b8 100755 --- a/emcc.py +++ b/emcc.py @@ -1483,7 +1483,8 @@ def check(input_file): shared.Settings.UBSAN_RUNTIME = 2 if 'leak' in sanitize: - shared.Settings.USE_LSAN = True + shared.Settings.USE_LSAN = 1 + shared.Settings.EXIT_RUNTIME = 1 if sanitize and '-g4' in args: shared.Settings.LOAD_SOURCE_MAP = 1 diff --git a/src/library_pthread_stub.js b/src/library_pthread_stub.js index 7ae3529c9bdde..7b25362e1c001 100644 --- a/src/library_pthread_stub.js +++ b/src/library_pthread_stub.js @@ -79,6 +79,10 @@ var LibraryPThreadStub = { {{{ makeSetValue('stacksize', '0', 'TOTAL_STACK', 'i32') }}}; return 0; }, + pthread_attr_getdetachstate: function(attr, detachstate) { + /* int pthread_attr_getdetachstate(const pthread_attr_t *attr, int *detachstate); */ + return 0; + }, pthread_setcancelstate: function() { return 0; }, pthread_setcanceltype: function() { return 0; }, diff --git a/system/lib/compiler-rt/lib/interception/interception.h b/system/lib/compiler-rt/lib/interception/interception.h index f80765a7ea654..d9eb53885e72a 100644 --- a/system/lib/compiler-rt/lib/interception/interception.h +++ b/system/lib/compiler-rt/lib/interception/interception.h @@ -132,7 +132,7 @@ const interpose_substitution substitution_##func_name[] \ extern "C" ret_type func(__VA_ARGS__); # define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \ extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__); -#elif SANITIZER_RTEMS +#elif SANITIZER_RTEMS || SANITIZER_EMSCRIPTEN # define WRAP(x) x # define WRAPPER_NAME(x) #x # define INTERCEPTOR_ATTRIBUTE diff --git a/system/lib/compiler-rt/lib/lsan/lsan_allocator.cc b/system/lib/compiler-rt/lib/lsan/lsan_allocator.cc index 1b338bd5973e8..9454e6a6877a2 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_allocator.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_allocator.cc @@ -27,7 +27,7 @@ extern "C" void *memset(void *ptr, int value, uptr num); namespace __lsan { -#if defined(__i386__) || defined(__arm__) +#if defined(__i386__) || defined(__arm__) || defined(__wasm32__) static const uptr kMaxAllowedMallocSize = 1UL << 30; #elif defined(__mips64) || defined(__aarch64__) static const uptr kMaxAllowedMallocSize = 4UL << 30; diff --git a/system/lib/compiler-rt/lib/lsan/lsan_allocator.h b/system/lib/compiler-rt/lib/lsan/lsan_allocator.h index 4c4e02fc09026..4965cf5670569 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_allocator.h +++ b/system/lib/compiler-rt/lib/lsan/lsan_allocator.h @@ -51,7 +51,7 @@ struct ChunkMetadata { }; #if defined(__mips64) || defined(__aarch64__) || defined(__i386__) || \ - defined(__arm__) + defined(__arm__) || defined(__wasm32__) static const uptr kRegionSizeLog = 20; static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; template diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.cc b/system/lib/compiler-rt/lib/lsan/lsan_common.cc index eaa5cadc8ffbc..e94eb07a20e85 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.cc @@ -323,14 +323,22 @@ void ScanRootRegion(Frontier *frontier, const RootRegion &root_region, kReachable); } +#if SANITIZER_EMSCRIPTEN +extern "C" uptr emscripten_get_heap_size(); +#endif + static void ProcessRootRegion(Frontier *frontier, const RootRegion &root_region) { +#if SANITIZER_EMSCRIPTEN + ScanRootRegion(frontier, root_region, 0, emscripten_get_heap_size(), true); +#else MemoryMappingLayout proc_maps(/*cache_enabled*/ true); MemoryMappedSegment segment; while (proc_maps.Next(&segment)) { ScanRootRegion(frontier, root_region, segment.start, segment.end, segment.IsReadable()); } +#endif // SANITIZER_EMSCRIPTEN } // Scans root regions for heap pointers. @@ -556,8 +564,10 @@ static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads, CheckForLeaksParam *param = reinterpret_cast(arg); CHECK(param); CHECK(!param->success); +#if !SANITIZER_EMSCRIPTEN ReportUnsuspendedThreads(suspended_threads); ClassifyAllChunks(suspended_threads); +#endif ForEachChunk(CollectLeaksCb, ¶m->leak_report); // Clean up for subsequent leak checks. This assumes we did not overwrite any // kIgnored tags. @@ -586,7 +596,7 @@ static bool CheckForLeaks() { "HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)\n"); Die(); } - param.leak_report.ApplySuppressions(); + //param.leak_report.ApplySuppressions(); uptr unsuppressed_count = param.leak_report.UnsuppressedLeakCount(); if (unsuppressed_count > 0) { Decorator d; diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.h b/system/lib/compiler-rt/lib/lsan/lsan_common.h index 1d1e1e4624357..556ccbc36a683 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.h +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.h @@ -41,6 +41,8 @@ #elif defined(__arm__) && \ SANITIZER_LINUX && !SANITIZER_ANDROID #define CAN_SANITIZE_LEAKS 1 +#elif SANITIZER_EMSCRIPTEN +#define CAN_SANITIZE_LEAKS 1 #else #define CAN_SANITIZE_LEAKS 0 #endif diff --git a/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc b/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc new file mode 100644 index 0000000000000..1fb7a193536d5 --- /dev/null +++ b/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc @@ -0,0 +1,103 @@ +//=-- lsan_common_linux.cc ------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of LeakSanitizer. +// Implementation of common leak checking functionality. Linux-specific code. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#include "lsan_common.h" + +#if CAN_SANITIZE_LEAKS && SANITIZER_EMSCRIPTEN +#include + +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_getauxval.h" +#include "sanitizer_common/sanitizer_linux.h" +#include "sanitizer_common/sanitizer_stackdepot.h" + + +namespace __lsan { + +extern "C" uptr emscripten_get_heap_size(); + +static const char kLinkerName[] = "ld"; + +static char linker_placeholder[sizeof(LoadedModule)] ALIGNED(64); +static LoadedModule *linker = nullptr; + +static bool IsLinker(const LoadedModule& module) { + return false; +} + +__attribute__((tls_model("initial-exec"))) +THREADLOCAL int disable_counter; +bool DisabledInThisThread() { return disable_counter > 0; } +void DisableInThisThread() { disable_counter++; } +void EnableInThisThread() { + if (disable_counter == 0) { + DisableCounterUnderflow(); + } + disable_counter--; +} + +void InitializePlatformSpecificModules() { + ListOfModules modules; + modules.init(); + for (LoadedModule &module : modules) { + if (!IsLinker(module)) + continue; + if (linker == nullptr) { + linker = reinterpret_cast(linker_placeholder); + *linker = module; + module = LoadedModule(); + } else { + VReport(1, "LeakSanitizer: Multiple modules match \"%s\". " + "TLS and other allocations originating from linker might be " + "falsely reported as leaks.\n", kLinkerName); + linker->clear(); + linker = nullptr; + return; + } + } + if (linker == nullptr) { + VReport(1, "LeakSanitizer: Dynamic linker not found. TLS and other " + "allocations originating from linker might be falsely reported " + "as leaks.\n"); + } +} + +// Scans global variables for heap pointers. +void ProcessGlobalRegions(Frontier *frontier) { + if (!flags()->use_globals) return; + ScanGlobalRange(0, emscripten_get_heap_size(), frontier); +} + +LoadedModule *GetLinker() { return linker; } + +void ProcessPlatformSpecificAllocations(Frontier *frontier) {} + +// While calling Die() here is undefined behavior and can potentially +// cause race conditions, it isn't possible to intercept exit on linux, +// so we have no choice but to call Die() from the atexit handler. +void HandleLeaks() { + if (common_flags()->exitcode) Die(); +} + +void DoStopTheWorld(StopTheWorldCallback callback, void *argument) { + StopTheWorld(callback, argument); +} + +void InitializeInterceptors() {} + +} // namespace __lsan + +#endif // CAN_SANITIZE_LEAKS && SANITIZER_LINUX diff --git a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cc b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cc index a9bd2ba42319e..aae184f144c81 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cc @@ -420,6 +420,7 @@ INTERCEPTOR(void, _exit, int status) { REAL(_exit)(status); } +#if !SANITIZER_EMSCRIPTEN #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) #include "sanitizer_common/sanitizer_signal_interceptors.inc" @@ -458,3 +459,4 @@ void InitializeInterceptors() { } } // namespace __lsan +#endif diff --git a/system/lib/compiler-rt/lib/lsan/lsan_linux.cc b/system/lib/compiler-rt/lib/lsan/lsan_linux.cc index c9749c7456551..1e46d5d08e4ce 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_linux.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_linux.cc @@ -13,7 +13,7 @@ #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_LINUX +#if SANITIZER_LINUX || SANITIZER_EMSCRIPTEN #include "lsan_allocator.h" diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc index 0fdd6a6654573..c58b1bbbadf8a 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc @@ -15,6 +15,7 @@ #include "sanitizer_platform.h" #include "sanitizer_common.h" +#include "sanitizer_stoptheworld.h" #include @@ -86,6 +87,21 @@ char **GetEnviron() { return fake_envp; } +uptr GetTlsSize() { + return 0; +} + +void InitTlsSize() { } + +void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, + uptr *tls_addr, uptr *tls_size) { + *stk_addr = *stk_size = *tls_addr = *tls_size = 0; +} + +void StopTheWorld(StopTheWorldCallback callback, void *argument) { + callback(SuspendedThreadsList(), argument); +} + } // namespace __sanitizer #endif diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc index 7e002ed9ad5f3..c14fc2ac3ad79 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc @@ -484,6 +484,8 @@ tid_t GetTid() { return internal_syscall(SYSCALL(getthrid)); #elif SANITIZER_SOLARIS return thr_self(); +#elif SANITIZER_EMSCRIPTEN + return (tid_t) pthread_self(); #else return internal_syscall(SYSCALL(gettid)); #endif diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc index 5abe3e9ceb761..89cda32f3a8cd 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc @@ -219,7 +219,9 @@ void InstallDeadlySignalHandlers(SignalHandlerType handler) { // Set the alternate signal stack for the main thread. // This will cause SetAlternateSignalStack to be called twice, but the stack // will be actually set only once. +#if !SANITIZER_EMSCRIPTEN if (common_flags()->use_sigaltstack) SetAlternateSignalStack(); +#endif MaybeInstallSigaction(SIGSEGV, handler); MaybeInstallSigaction(SIGBUS, handler); MaybeInstallSigaction(SIGABRT, handler); From 69ff180010526bcd569b705992c876e26e55547b Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Fri, 31 May 2019 11:25:46 -0700 Subject: [PATCH 07/62] Working global/stack leak detection --- system/lib/compiler-rt/lib/lsan/lsan_common.cc | 17 +++++++++++++---- system/lib/compiler-rt/lib/lsan/lsan_common.h | 4 ++++ .../lib/compiler-rt/lib/lsan/lsan_emscripten.cc | 13 ++++++++++++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.cc b/system/lib/compiler-rt/lib/lsan/lsan_common.cc index e94eb07a20e85..59ada5c0be132 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.cc @@ -212,9 +212,10 @@ void ForEachExtraStackRangeCb(uptr begin, uptr end, void* arg) { ScanRangeForPointers(begin, end, frontier, "FAKE STACK", kReachable); } +#if !SANITIZER_EMSCRIPTEN // Scans thread data (stacks and TLS) for heap pointers. -static void ProcessThreads(SuspendedThreadsList const &suspended_threads, - Frontier *frontier) { +void ProcessThreads(SuspendedThreadsList const &suspended_threads, + Frontier *frontier) { InternalMmapVector registers(suspended_threads.RegisterCount()); uptr registers_begin = reinterpret_cast(registers.data()); uptr registers_end = @@ -308,6 +309,7 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads, } } } +#endif // !SANITIZER_EMSCRIPTEN void ScanRootRegion(Frontier *frontier, const RootRegion &root_region, uptr region_begin, uptr region_end, bool is_readable) { @@ -442,6 +444,7 @@ static void MarkInvalidPCCb(uptr chunk, void *arg) { // On all other platforms, this simply checks to ensure that the caller pc is // valid before reporting chunks as leaked. void ProcessPC(Frontier *frontier) { +#if !SANITIZER_EMSCRIPTEN StackDepotReverseMap stack_depot_reverse_map; InvalidPCParam arg; arg.frontier = frontier; @@ -449,6 +452,7 @@ void ProcessPC(Frontier *frontier) { arg.skip_linker_allocations = flags()->use_tls && flags()->use_ld_allocations && GetLinker() != nullptr; ForEachChunk(MarkInvalidPCCb, &arg); +#endif } // Sets the appropriate tag on each chunk. @@ -566,8 +570,8 @@ static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads, CHECK(!param->success); #if !SANITIZER_EMSCRIPTEN ReportUnsuspendedThreads(suspended_threads); - ClassifyAllChunks(suspended_threads); #endif + ClassifyAllChunks(suspended_threads); ForEachChunk(CollectLeaksCb, ¶m->leak_report); // Clean up for subsequent leak checks. This assumes we did not overwrite any // kIgnored tags. @@ -596,8 +600,11 @@ static bool CheckForLeaks() { "HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)\n"); Die(); } - //param.leak_report.ApplySuppressions(); +#if !SANITIZER_EMSCRIPTEN + param.leak_report.ApplySuppressions(); +#endif uptr unsuppressed_count = param.leak_report.UnsuppressedLeakCount(); + if (unsuppressed_count > 0) { Decorator d; Printf("\n" @@ -743,7 +750,9 @@ void LeakReport::PrintReportForLeak(uptr index) { leaks_[index].total_size, leaks_[index].hit_count); Printf("%s", d.Default()); +#if !SANITIZER_EMSCRIPTEN PrintStackTraceById(leaks_[index].stack_trace_id); +#endif if (flags()->report_objects) { Printf("Objects leaked above:\n"); diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.h b/system/lib/compiler-rt/lib/lsan/lsan_common.h index 556ccbc36a683..c79099f8a703c 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.h +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.h @@ -213,6 +213,10 @@ bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end, uptr *cache_end, DTLS **dtls); void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback, void *arg); +// Scans thread data (stacks and TLS) for heap pointers. +void ProcessThreads(SuspendedThreadsList const &suspended_threads, + Frontier *frontier); + // If called from the main thread, updates the main thread's TID in the thread // registry. We need this to handle processes that fork() without a subsequent // exec(), which invalidates the recorded TID. To update it, we must call diff --git a/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc b/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc index 1fb7a193536d5..c89a916ae513d 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc @@ -75,10 +75,15 @@ void InitializePlatformSpecificModules() { } } +extern "C" { + extern int __data_end; + extern int __heap_base; +} + // Scans global variables for heap pointers. void ProcessGlobalRegions(Frontier *frontier) { if (!flags()->use_globals) return; - ScanGlobalRange(0, emscripten_get_heap_size(), frontier); + ScanGlobalRange(0, (uptr) &__data_end, frontier); } LoadedModule *GetLinker() { return linker; } @@ -98,6 +103,12 @@ void DoStopTheWorld(StopTheWorldCallback callback, void *argument) { void InitializeInterceptors() {} +void ProcessThreads(SuspendedThreadsList const &suspended_threads, + Frontier *frontier) { + uptr sp = (uptr) __builtin_frame_address(0); + ScanRangeForPointers(sp, (uptr) &__heap_base, frontier, "STACK", kReachable); +} + } // namespace __lsan #endif // CAN_SANITIZE_LEAKS && SANITIZER_LINUX From 1dee1659c8905d4aa4afcde53eb1d638bd5d39a5 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Fri, 31 May 2019 15:26:25 -0700 Subject: [PATCH 08/62] Use emscripten_builtin_malloc/free for stack trace functionality --- src/library.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/library.js b/src/library.js index a1ef61eac788c..41caf2d832dda 100644 --- a/src/library.js +++ b/src/library.js @@ -4530,8 +4530,8 @@ LibraryManager.library = { } if (!name) return 0; - if (_emscripten_pc_get_function.ret) _free(_emscripten_pc_get_function.ret); - _emscripten_pc_get_function.ret = allocateUTF8(name); + if (_emscripten_pc_get_function.ret) _emscripten_builtin_free(_emscripten_pc_get_function.ret); + _emscripten_pc_get_function.ret = allocateUTF8(name, _emscripten_builtin_malloc); return _emscripten_pc_get_function.ret; }, @@ -4572,8 +4572,8 @@ LibraryManager.library = { var result = _emscripten_pc_get_source_js(pc); if (!result) return 0; - if (_emscripten_pc_get_file.ret) _free(_emscripten_pc_get_file.ret); - _emscripten_pc_get_file.ret = allocateUTF8(result.file); + if (_emscripten_pc_get_file.ret) _emscripten_builtin_free(_emscripten_pc_get_file.ret); + _emscripten_pc_get_file.ret = allocateUTF8(result.file, _emscripten_builtin_malloc); return _emscripten_pc_get_file.ret; }, From 39dfd17fc039e2bada1a32a11eadf152b5fb5150 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Fri, 31 May 2019 15:29:50 -0700 Subject: [PATCH 09/62] Show LSan stack trace --- system/lib/compiler-rt/lib/lsan/lsan_common.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.cc b/system/lib/compiler-rt/lib/lsan/lsan_common.cc index 59ada5c0be132..1f914461f80bf 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.cc @@ -750,9 +750,7 @@ void LeakReport::PrintReportForLeak(uptr index) { leaks_[index].total_size, leaks_[index].hit_count); Printf("%s", d.Default()); -#if !SANITIZER_EMSCRIPTEN PrintStackTraceById(leaks_[index].stack_trace_id); -#endif if (flags()->report_objects) { Printf("Objects leaked above:\n"); From 5cbb27c5dabcb618f7e66f7e6765e4861e49324c Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 3 Jun 2019 16:07:12 -0700 Subject: [PATCH 10/62] Export __data_end and __heap_base when using lsan --- tools/system_libs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/system_libs.py b/tools/system_libs.py index a25ba334ed5c0..a0687936c2641 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -851,6 +851,7 @@ def do_create(): if shared.Settings.USE_LSAN: need_common_san = True libs_to_link.append((shared.Cache.get('liblsan_rt_wasm.a', lambda: create_wasm_lsan_rt('liblsan_rt_wasm.a')), True)) + shared.Settings.EXPORTED_FUNCTIONS += ['___data_end', '___heap_base'] if need_common_san: add_library(system_libs_map['libc++abi']) From 132f8462cbabf8f5cf46ecce0b5a20c351bbf0b0 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 3 Jun 2019 17:09:30 -0700 Subject: [PATCH 11/62] Make LSan not cache its own allocator cache --- .../lib/compiler-rt/lib/lsan/lsan_common.cc | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.cc b/system/lib/compiler-rt/lib/lsan/lsan_common.cc index 1f914461f80bf..9b4a8f7b5532c 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.cc @@ -26,6 +26,10 @@ #include "sanitizer_common/sanitizer_thread_registry.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" +#if SANITIZER_EMSCRIPTEN +#include "lsan/lsan_allocator.h" +#endif + #if CAN_SANITIZE_LEAKS namespace __lsan { @@ -163,6 +167,15 @@ void ScanRangeForPointers(uptr begin, uptr end, uptr pp = begin; if (pp % alignment) pp = pp + alignment - pp % alignment; + + // TODO: Emscripten doesn't support thread local storage, + // so the cache is in the global range. + // Remove this when Emscripten has TLS implemented. +#if SANITIZER_EMSCRIPTEN + uptr cache_begin, cache_end; + GetAllocatorCacheRange(&cache_begin, &cache_end); +#endif + for (; pp + sizeof(void *) <= end; pp += alignment) { // NOLINT void *p = *reinterpret_cast(pp); if (!CanBeAHeapPointer(reinterpret_cast(p))) continue; @@ -182,6 +195,14 @@ void ScanRangeForPointers(uptr begin, uptr end, continue; } +#if SANITIZER_EMSCRIPTEN + if (cache_begin <= pp || pp < cache_end) { + LOG_POINTERS("%p: skipping because it overlaps the cache %p-%p.\n", + pp, cache_begin, cache_end); + continue; + } +#endif + m.set_tag(tag); LOG_POINTERS("%p: found %p pointing into chunk %p-%p of size %zu.\n", pp, p, chunk, chunk + m.requested_size(), m.requested_size()); From 8a071124c59366f49eb3cbbb0f055c44583cedb5 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Tue, 4 Jun 2019 13:21:44 -0700 Subject: [PATCH 12/62] Fix library dependencies --- src/library.js | 2 ++ src/library_syscall.js | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/library.js b/src/library.js index 41caf2d832dda..2254a80036566 100644 --- a/src/library.js +++ b/src/library.js @@ -817,11 +817,13 @@ LibraryManager.library = { var ptr = dynamicAlloc(bytes + 8); return (ptr+8) & 0xFFFFFFF8; }, + emscripten_builtin_malloc: 'malloc', free: function() { #if ASSERTIONS == 2 warnOnce('using stub free (reference it from C to have the real one included)'); #endif }, + emscripten_builtin_free: 'free', #endif abs: 'Math_abs', diff --git a/src/library_syscall.js b/src/library_syscall.js index aea529d981f4d..31212a6e132a3 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -444,7 +444,6 @@ var SyscallsLibrary = { var path = SYSCALLS.getStr(), buf = SYSCALLS.get(), bufsize = SYSCALLS.get(); return SYSCALLS.doReadlink(path, buf, bufsize); }, - __syscall91__deps: ['emscripten_builtin_free'], __syscall91: function(which, varargs) { // munmap var addr = SYSCALLS.get(), len = SYSCALLS.get(); // TODO: support unmmap'ing parts of allocations From 6b99edd61948cedc902fb612af154ad23ddf2aa5 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Tue, 4 Jun 2019 15:09:31 -0700 Subject: [PATCH 13/62] Fix fastcomp dlfcn tests --- src/preamble.js | 2 +- system/lib/dlmalloc.c | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index c228ad7729240..076391555b52c 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -261,7 +261,7 @@ function allocate(slab, types, allocator, ptr) { // Allocate memory during any stage of startup - static memory early on, dynamic memory later, malloc when ready function getMemory(size) { if (!runtimeInitialized) return dynamicAlloc(size); - return _malloc(size); + return _emscripten_builtin_malloc(size); } #include "runtime_strings.js" diff --git a/system/lib/dlmalloc.c b/system/lib/dlmalloc.c index 54ef05b2c2f8a..be375ab9ededd 100644 --- a/system/lib/dlmalloc.c +++ b/system/lib/dlmalloc.c @@ -1,7 +1,7 @@ /* XXX Emscripten XXX */ #if __EMSCRIPTEN__ -#define DLMALLOC_EXPORT __attribute__((__weak__)) +#define DLMALLOC_EXPORT extern /* mmap uses malloc, so malloc can't use mmap */ #define HAVE_MMAP 0 /* we can only grow the heap up anyhow, so don't try to trim */ @@ -851,10 +851,24 @@ extern "C" { #ifndef USE_DL_PREFIX // XXX Emscripten XXX - #ifdef __EMSCRIPTEN__ +// XXX fastcomp HACK: in fastcomp, weak aliases loses to JavaScript versions of the function. +// Therefore, we must define real functions. Remove this hack when fastcomp is removed. +#ifdef __asmjs__ +void* dlmalloc(size_t); +__attribute__((weak)) void* malloc(size_t size) { + return dlmalloc(size); +} + +void dlfree(void*); +__attribute__((weak)) void free(void *p) { + dlfree(p); +} +#else void* malloc(size_t) __attribute__((weak, alias("dlmalloc"))); void free(void*) __attribute__((weak, alias("dlfree"))); +#endif /* __asmjs__ */ + void* calloc(size_t, size_t) __attribute__((weak, alias("dlcalloc"))); void* realloc(void*, size_t) __attribute__((weak, alias("dlrealloc"))); void* realloc_in_place(void*, size_t) __attribute__((weak, alias("dlrealloc_in_place"))); @@ -862,6 +876,23 @@ void* memalign(size_t, size_t) __attribute__((weak, alias("dlmemalign"))); int posix_memalign(void**, size_t, size_t) __attribute__((weak, alias("dlposix_memalign"))); void* valloc(size_t) __attribute__((weak, alias("dlvalloc"))); void* pvalloc(size_t) __attribute__((weak, alias("dlpvalloc"))); +#if !NO_MALLINFO +struct mallinfo mallinfo(void) __attribute__((weak, alias("dlmallinfo"))); +#endif +int mallopt(int, int) __attribute__((weak, alias("dlmallopt"))); +int malloc_trim(size_t) __attribute__((weak, alias("dlmalloc_trim"))); +void malloc_stats(void) __attribute__((weak, alias("dlmalloc_stats"))); +size_t malloc_usable_size(const void*) __attribute__((weak, alias("dlmalloc_usable_size"))); +size_t malloc_footprint(void) __attribute__((weak, alias("dlmalloc_footprint"))); +size_t malloc_max_footprint(void) __attribute__((weak, alias("dlmalloc_max_footprint"))); +size_t malloc_footprint_limit(void) __attribute__((weak, alias("dlmalloc_footprint_limit"))); +size_t malloc_set_footprint_limit(size_t bytes) __attribute__((weak, alias("dlmalloc_set_footprint_limit"))); +#if MALLOC_INSPECT_ALL +void malloc_inspect_all(void(*handler)(void*, void *, size_t, void*), void* arg) __attribute__((weak, alias("dlmalloc_inspect_all"))); +#endif +void** independent_calloc(size_t, size_t, void**) __attribute__((weak, alias("dlindependent_calloc"))); +void** independent_comalloc(size_t, size_t*, void**) __attribute__((weak, alias("dlindependent_comalloc"))); +size_t bulk_free(void**, size_t n_elements) __attribute__((weak, alias("dlbulk_free"))); #else #define dlcalloc calloc #define dlfree free @@ -872,7 +903,6 @@ void* pvalloc(size_t) __attribute__((weak, alias("dlpvalloc"))); #define dlrealloc_in_place realloc_in_place #define dlvalloc valloc #define dlpvalloc pvalloc -#endif #define dlmallinfo mallinfo #define dlmallopt mallopt #define dlmalloc_trim malloc_trim @@ -886,6 +916,7 @@ void* pvalloc(size_t) __attribute__((weak, alias("dlpvalloc"))); #define dlindependent_calloc independent_calloc #define dlindependent_comalloc independent_comalloc #define dlbulk_free bulk_free +#endif #endif /* USE_DL_PREFIX */ /* From ee7642ea03ef92d4a38a5da546041ae76252876c Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Tue, 4 Jun 2019 15:31:58 -0700 Subject: [PATCH 14/62] Make test_dlmalloc work by defining dl* as weak again --- system/lib/dlmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/lib/dlmalloc.c b/system/lib/dlmalloc.c index be375ab9ededd..796dfb6a71f72 100644 --- a/system/lib/dlmalloc.c +++ b/system/lib/dlmalloc.c @@ -1,7 +1,7 @@ /* XXX Emscripten XXX */ #if __EMSCRIPTEN__ -#define DLMALLOC_EXPORT extern +#define DLMALLOC_EXPORT __attribute__((weak)) /* mmap uses malloc, so malloc can't use mmap */ #define HAVE_MMAP 0 /* we can only grow the heap up anyhow, so don't try to trim */ From eed8741463108455f95017f24e1028ff6c923dfd Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Tue, 4 Jun 2019 17:17:22 -0700 Subject: [PATCH 15/62] Don't try to install signal handlers on emscripten --- system/lib/compiler-rt/lib/lsan/lsan.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/system/lib/compiler-rt/lib/lsan/lsan.cc b/system/lib/compiler-rt/lib/lsan/lsan.cc index 93bced0459c25..8e9f3cf1904da 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan.cc @@ -96,7 +96,10 @@ extern "C" void __lsan_init() { InitTlsSize(); InitializeInterceptors(); InitializeThreadRegistry(); +#if !SANITIZER_EMSCRIPTEN + // Emscripten does not have signals InstallDeadlySignalHandlers(LsanOnDeadlySignal); +#endif u32 tid = ThreadCreate(0, 0, true); CHECK_EQ(tid, 0); ThreadStart(tid, GetTid()); From e87da4c386551c0cc4b30475ac785f610562dfef Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Tue, 4 Jun 2019 17:26:27 -0700 Subject: [PATCH 16/62] Implement LSan suppressions --- system/lib/compiler-rt/lib/lsan/lsan_common.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.cc b/system/lib/compiler-rt/lib/lsan/lsan_common.cc index 9b4a8f7b5532c..1da59be571e0f 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.cc @@ -621,9 +621,7 @@ static bool CheckForLeaks() { "HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)\n"); Die(); } -#if !SANITIZER_EMSCRIPTEN param.leak_report.ApplySuppressions(); -#endif uptr unsuppressed_count = param.leak_report.UnsuppressedLeakCount(); if (unsuppressed_count > 0) { @@ -690,8 +688,15 @@ static Suppression *GetSuppressionForAddr(uptr addr) { static Suppression *GetSuppressionForStack(u32 stack_trace_id) { StackTrace stack = StackDepotGet(stack_trace_id); for (uptr i = 0; i < stack.size; i++) { +#if SANITIZER_EMSCRIPTEN + // On Emscripten, the stack trace is the actual call site, not + // the code that would be executed after the return. + // THerefore, StackTrace::GetPreviousInstructionPc is not needed. + Suppression *s = GetSuppressionForAddr(stack.trace[i]); +#else Suppression *s = GetSuppressionForAddr( StackTrace::GetPreviousInstructionPc(stack.trace[i])); +#endif if (s) return s; } return nullptr; From cb02ce73d0e3f510b125b286b5107603b06c66e2 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 5 Jun 2019 11:00:06 -0700 Subject: [PATCH 17/62] Fix sdl2-mixer fastcomp build --- src/preamble.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/preamble.js b/src/preamble.js index 076391555b52c..c228ad7729240 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -261,7 +261,7 @@ function allocate(slab, types, allocator, ptr) { // Allocate memory during any stage of startup - static memory early on, dynamic memory later, malloc when ready function getMemory(size) { if (!runtimeInitialized) return dynamicAlloc(size); - return _emscripten_builtin_malloc(size); + return _malloc(size); } #include "runtime_strings.js" From 9bba5d90f735115c8100c7fbb914209574c2f582 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 5 Jun 2019 11:01:28 -0700 Subject: [PATCH 18/62] Run original dlmalloc on fastcomp --- system/lib/dlmalloc.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/system/lib/dlmalloc.c b/system/lib/dlmalloc.c index 796dfb6a71f72..a34c5b9adf89f 100644 --- a/system/lib/dlmalloc.c +++ b/system/lib/dlmalloc.c @@ -851,24 +851,11 @@ extern "C" { #ifndef USE_DL_PREFIX // XXX Emscripten XXX -#ifdef __EMSCRIPTEN__ +#if defined(__EMSCRIPTEN__) && !defined(__asmjs__) // XXX fastcomp HACK: in fastcomp, weak aliases loses to JavaScript versions of the function. // Therefore, we must define real functions. Remove this hack when fastcomp is removed. -#ifdef __asmjs__ -void* dlmalloc(size_t); -__attribute__((weak)) void* malloc(size_t size) { - return dlmalloc(size); -} - -void dlfree(void*); -__attribute__((weak)) void free(void *p) { - dlfree(p); -} -#else void* malloc(size_t) __attribute__((weak, alias("dlmalloc"))); void free(void*) __attribute__((weak, alias("dlfree"))); -#endif /* __asmjs__ */ - void* calloc(size_t, size_t) __attribute__((weak, alias("dlcalloc"))); void* realloc(void*, size_t) __attribute__((weak, alias("dlrealloc"))); void* realloc_in_place(void*, size_t) __attribute__((weak, alias("dlrealloc_in_place"))); @@ -6077,9 +6064,15 @@ int mspace_mallopt(int param_number, int value) { // and dlfree from this file. // This allows an easy mechanism for hooking into memory allocation. #if defined(__EMSCRIPTEN__) && !ONLY_MSPACES +#ifdef __asmjs__ +extern __typeof(malloc) emscripten_builtin_malloc __attribute__((alias("malloc"))); +extern __typeof(free) emscripten_builtin_free __attribute__((alias("free"))); +extern __typeof(memalign) emscripten_builtin_memalign __attribute__((alias("memalign"))); +#else extern __typeof(malloc) emscripten_builtin_malloc __attribute__((alias("dlmalloc"))); extern __typeof(free) emscripten_builtin_free __attribute__((alias("dlfree"))); extern __typeof(memalign) emscripten_builtin_memalign __attribute__((alias("dlmemalign"))); +#endif /* __asmjs__ */ #endif /* -------------------- Alternative MORECORE functions ------------------- */ From 145f8a58321c207eeecaa3ebd9e32f20c30ddbc9 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 5 Jun 2019 11:38:15 -0700 Subject: [PATCH 19/62] Keep using _malloc and _free on fastcomp --- src/deps_info.json | 6 +++--- src/library_syscall.js | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/deps_info.json b/src/deps_info.json index 49447e06b6498..c03e6aabb10bb 100644 --- a/src/deps_info.json +++ b/src/deps_info.json @@ -1,7 +1,7 @@ { "arc4random": ["rand"], "freopen": ["free"], - "munmap": ["emscripten_builtin_free"], + "munmap": ["emscripten_builtin_free", "free"], "getenv": ["emscripten_builtin_malloc", "emscripten_builtin_free", "_get_environ"], "clearenv": ["_get_environ"], "setenv": ["_get_environ"], @@ -19,7 +19,7 @@ "realloc": ["malloc", "free"], "getlogin": ["malloc", "_get_environ"], "tmpnam": ["malloc"], - "mmap": ["emscripten_builtin_memalign"], + "mmap": ["emscripten_builtin_memalign", "memalign"], "realpath": ["malloc"], "strerror": ["malloc"], "__ctype_b_loc": ["malloc"], @@ -89,6 +89,6 @@ "stringToNewUTF8": ["malloc"], "_embind_register_std_string": ["malloc", "free"], "_embind_register_std_wstring": ["malloc", "free"], - "__syscall192": ["memalign"] + "__syscall192": ["emscripten_builtin_memalign"] } diff --git a/src/library_syscall.js b/src/library_syscall.js index 31212a6e132a3..eb6da1bfd6be2 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -455,7 +455,11 @@ var SyscallsLibrary = { FS.munmap(stream); SYSCALLS.mappings[addr] = null; if (info.allocated) { +#if WASM_BACKEND _emscripten_builtin_free(info.malloc); +#else + _free(info.malloc); +#endif } } return 0; @@ -982,7 +986,11 @@ var SyscallsLibrary = { var ptr; var allocated = false; if (fd === -1) { +#if WASM_BACKEND ptr = _emscripten_builtin_memalign(PAGE_SIZE, len); +#else + ptr = _memalign(PAGE_SIZE, len); +#endif if (!ptr) return -{{{ cDefine('ENOMEM') }}}; _memset(ptr, 0, len); allocated = true; From 0cdff24505924c461fdb5b377b65434bf6544c60 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 5 Jun 2019 12:54:27 -0700 Subject: [PATCH 20/62] Update metadce expectations --- tests/other/metadce/hello_world_O3_MAIN_MODULE.sent | 1 + tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent | 4 ++++ tests/test_other.py | 4 ++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent b/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent index 0088d687f5872..8f59a5e7a9d9f 100644 --- a/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent +++ b/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent @@ -1475,6 +1475,7 @@ pixelRGBA posix_spawn posix_spawnp pthread_attr_destroy +pthread_attr_getdetachstate pthread_attr_getstack pthread_attr_init pthread_attr_setdetachstate diff --git a/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent b/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent index 40fe97fed6048..679ad204f7316 100644 --- a/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent +++ b/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent @@ -638,6 +638,9 @@ _emscripten_autodebug_i16 _emscripten_autodebug_i32 _emscripten_autodebug_i64 _emscripten_autodebug_i8 +_emscripten_builtin_free +_emscripten_builtin_malloc +_emscripten_builtin_memalign _emscripten_call_worker _emscripten_cancel_animation_frame _emscripten_cancel_main_loop @@ -1422,6 +1425,7 @@ _pixelRGBA _posix_spawn _posix_spawnp _pthread_attr_destroy +_pthread_attr_getdetachstate _pthread_attr_getstack _pthread_attr_init _pthread_attr_setdetachstate diff --git a/tests/test_other.py b/tests/test_other.py index 12e06fe4bad41..e51f137c286f5 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7969,7 +7969,7 @@ def test_binaryen_metadce_cxx_fastcomp(self, *args): # don't compare the # of functions in a main module, which changes a lot # TODO(sbc): Investivate why the number of exports is order of magnitude # larger for wasm backend. - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1604, [], [], 517336, 172, 1484, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1605, [], [], 517336, 172, 1484, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10770, 17, 13, None), # noqa }) @no_fastcomp() @@ -7989,7 +7989,7 @@ def test_binaryen_metadce_hello(self, *args): 0, [], [], 8, 0, 0, 0), # noqa; totally empty! # we don't metadce with linkable code! other modules may want stuff # don't compare the # of functions in a main module, which changes a lot - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1566, [], [], 226403, 30, 96, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1570, [], [], 226403, 30, 96, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10571, 19, 9, 21), # noqa }) @no_wasm_backend() From 1ea40152a591df03821b125d807c07d02461cccc Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 5 Jun 2019 13:22:09 -0700 Subject: [PATCH 21/62] Fix various problems --- src/deps_info.json | 4 ++-- src/library.js | 5 +++++ src/library_syscall.js | 2 +- system/lib/compiler-rt/lib/lsan/lsan_common.cc | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/deps_info.json b/src/deps_info.json index c03e6aabb10bb..5cf2ca0c9aecc 100644 --- a/src/deps_info.json +++ b/src/deps_info.json @@ -2,7 +2,7 @@ "arc4random": ["rand"], "freopen": ["free"], "munmap": ["emscripten_builtin_free", "free"], - "getenv": ["emscripten_builtin_malloc", "emscripten_builtin_free", "_get_environ"], + "getenv": ["emscripten_builtin_malloc", "emscripten_builtin_free", "malloc", "free", "_get_environ"], "clearenv": ["_get_environ"], "setenv": ["_get_environ"], "unsetenv": ["_get_environ"], @@ -10,7 +10,7 @@ "environ": ["_get_environ"], "fmtmsg": ["_get_environ"], "get_current_dir_name": ["_get_environ"], - "SDL_getenv": ["emscripten_builtin_malloc", "emscripten_builtin_free", "_get_environ"], + "SDL_getenv": ["emscripten_builtin_malloc", "emscripten_builtin_free", "malloc", "free", "_get_environ"], "SDL_putenv": ["_get_environ"], "dlerror": ["malloc", "free"], "readdir": ["malloc"], diff --git a/src/library.js b/src/library.js index 2254a80036566..133621cd1ab33 100644 --- a/src/library.js +++ b/src/library.js @@ -933,8 +933,13 @@ LibraryManager.library = { name = UTF8ToString(name); if (!ENV.hasOwnProperty(name)) return 0; +#if WASM_BACKEND if (_getenv.ret) _emscripten_builtin_free(_getenv.ret); _getenv.ret = allocateUTF8(ENV[name], _emscripten_builtin_malloc); +#else + if (_getenv.ret) _free(_getenv.ret); + _getenv.ret = allocateUTF8(ENV[name]); +#endif return _getenv.ret; }, // Alias for sanitizers which intercept getenv. diff --git a/src/library_syscall.js b/src/library_syscall.js index eb6da1bfd6be2..68b337812069f 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -979,7 +979,7 @@ var SyscallsLibrary = { {{{ makeSetValue('rlim', C_STRUCTS.rlimit.rlim_max + 4, '-1', 'i32') }}}; // RLIM_INFINITY return 0; // just report no limits }, - __syscall192__deps: ['emscripten_builtin_memalign'], + __syscall192__deps: ['emscripten_builtin_memalign', 'memalign'], __syscall192: function(which, varargs) { // mmap2 var addr = SYSCALLS.get(), len = SYSCALLS.get(), prot = SYSCALLS.get(), flags = SYSCALLS.get(), fd = SYSCALLS.get(), off = SYSCALLS.get() off <<= 12; // undo pgoffset diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.cc b/system/lib/compiler-rt/lib/lsan/lsan_common.cc index 1da59be571e0f..257fa7a156c6b 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.cc @@ -691,7 +691,7 @@ static Suppression *GetSuppressionForStack(u32 stack_trace_id) { #if SANITIZER_EMSCRIPTEN // On Emscripten, the stack trace is the actual call site, not // the code that would be executed after the return. - // THerefore, StackTrace::GetPreviousInstructionPc is not needed. + // Therefore, StackTrace::GetPreviousInstructionPc is not needed. Suppression *s = GetSuppressionForAddr(stack.trace[i]); #else Suppression *s = GetSuppressionForAddr( From 636141af7602fe03420770c4a7d45d4b6c796b21 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 5 Jun 2019 14:00:36 -0700 Subject: [PATCH 22/62] Fix memalign not found --- tools/system_libs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index a0687936c2641..083f60bf53b47 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -856,7 +856,7 @@ def do_create(): if need_common_san: add_library(system_libs_map['libc++abi']) libs_to_link.append((shared.Cache.get('libsanitizer_common_rt_wasm.a', lambda: create_wasm_common_san_rt('libsanitizer_common_rt_wasm.a')), False)) - shared.Settings.EXPORTED_FUNCTIONS += ['_emscripten_builtin_memalign'] + shared.Settings.EXPORTED_FUNCTIONS += ['_emscripten_builtin_memalign', '_memalign'] libs_to_link.sort(key=lambda x: x[0].endswith('.a')) # make sure to put .a files at the end. From 85d507667fab85300d7d7be898ab1d63ab453893 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 5 Jun 2019 14:12:50 -0700 Subject: [PATCH 23/62] Fix metadce tests --- emcc.py | 4 +- .../metadce/hello_libcxx_fastcomp_O2.sent | 2 + ...astcomp_O2_EMULATED_FUNCTION_POINTERS.sent | 2 + tests/test_other.py | 40 +++++++++++-------- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/emcc.py b/emcc.py index b3d47884737b8..87e426944dac8 100755 --- a/emcc.py +++ b/emcc.py @@ -1199,11 +1199,11 @@ def check(input_file): if not shared.Settings.ONLY_MY_CODE and not shared.Settings.MINIMAL_RUNTIME: # Always need malloc and free to be kept alive and exported, for internal use and other modules - shared.Settings.EXPORTED_FUNCTIONS += ['_malloc', '_free', '_emscripten_builtin_malloc', '_emscripten_builtin_free'] + shared.Settings.EXPORTED_FUNCTIONS += ['_malloc', '_free'] if shared.Settings.WASM_BACKEND: # setjmp/longjmp and exception handling JS code depends on this so we # include it by default. Should be eliminated by meta-DCE if unused. - shared.Settings.EXPORTED_FUNCTIONS += ['_setThrew'] + shared.Settings.EXPORTED_FUNCTIONS += ['_setThrew', '_emscripten_builtin_malloc', '_emscripten_builtin_free'] if shared.Settings.RELOCATABLE and not shared.Settings.DYNAMIC_EXECUTION: exit_with_error('cannot have both DYNAMIC_EXECUTION=0 and RELOCATABLE enabled at the same time, since RELOCATABLE needs to eval()') diff --git a/tests/other/metadce/hello_libcxx_fastcomp_O2.sent b/tests/other/metadce/hello_libcxx_fastcomp_O2.sent index 2de6935c7b52e..0f041331b5393 100644 --- a/tests/other/metadce/hello_libcxx_fastcomp_O2.sent +++ b/tests/other/metadce/hello_libcxx_fastcomp_O2.sent @@ -15,6 +15,8 @@ __addDays __arraySum __isLeapYear _abort +_emscripten_builtin_free +_emscripten_builtin_malloc _emscripten_get_heap_size _emscripten_memcpy_big _emscripten_resize_heap diff --git a/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent b/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent index 2de6935c7b52e..0f041331b5393 100644 --- a/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent +++ b/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent @@ -15,6 +15,8 @@ __addDays __arraySum __isLeapYear _abort +_emscripten_builtin_free +_emscripten_builtin_malloc _emscripten_get_heap_size _emscripten_memcpy_big _emscripten_resize_heap diff --git a/tests/test_other.py b/tests/test_other.py index e51f137c286f5..1b8dc05b62716 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7629,12 +7629,20 @@ def test_binaryen_names(self): # we use dlmalloc here, as emmalloc has a bunch of asserts that contain the text "malloc" in them, which makes counting harder run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp')] + args + ['-s', 'MALLOC="dlmalloc"']) code = open('a.out.wasm', 'rb').read() - if expect_names: - # name section adds the name of malloc (there is also another one for the export) - self.assertEqual(code.count(b'malloc'), 2) + if self.is_wasm_backend(): + if expect_names: + # name section adds the name of malloc (there is also another one for the exports: malloc and emscripten_builtin_malloc) + self.assertEqual(code.count(b'malloc'), 3) + else: + # should be just malloc and emscripten_builtin_malloc, for the export + self.assertEqual(code.count(b'malloc'), 2) else: - # should be just one name, for the export - self.assertEqual(code.count(b'malloc'), 1) + if expect_names: + # name section adds the name of malloc (there is also another one for the export) + self.assertEqual(code.count(b'malloc'), 2) + else: + # should be just malloc for the export + self.assertEqual(code.count(b'malloc'), 1) sizes[str(args)] = os.path.getsize('a.out.wasm') print(sizes) self.assertLess(sizes["['-O2']"], sizes["['-O2', '--profiling-funcs']"], 'when -profiling-funcs, the size increases due to function names') @@ -7913,9 +7921,9 @@ def args_to_filename(args): self.assertEqual(funcs, expected_funcs) @parameterized({ - 'O0': ([], 11, [], ['waka'], 9211, 5, 12, 16), # noqa - 'O1': (['-O1'], 9, [], ['waka'], 7886, 2, 11, 10), # noqa - 'O2': (['-O2'], 9, [], ['waka'], 7871, 2, 11, 10), # noqa + 'O0': ([], 11, [], ['waka'], 9211, 5, 14, 16), # noqa + 'O1': (['-O1'], 9, [], ['waka'], 7886, 2, 13, 10), # noqa + 'O2': (['-O2'], 9, [], ['waka'], 7871, 2, 13, 10), # noqa # in -O3, -Os and -Oz we metadce, and they shrink it down to the minimal output we want 'O3': (['-O3'], 0, [], [], 85, 0, 2, 2), # noqa 'Os': (['-Os'], 0, [], [], 85, 0, 2, 2), # noqa @@ -7941,13 +7949,13 @@ def test_binaryen_metadce_minimal_fastcomp(self, *args): @no_fastcomp() def test_binaryen_metadce_cxx(self): # test on libc++: see effects of emulated function pointers - self.run_metadce_test('hello_libcxx.cpp', ['-O2'], 32, [], ['waka'], 226582, 21, 32, 559) # noqa + self.run_metadce_test('hello_libcxx.cpp', ['-O2'], 32, [], ['waka'], 226582, 21, 34, 559) # noqa @parameterized({ - 'normal': (['-O2'], 31, ['abort'], ['waka'], 186423, 29, 38, 539), # noqa + 'normal': (['-O2'], 33, ['abort'], ['waka'], 186423, 29, 38, 539), # noqa 'enumated_function_pointers': (['-O2', '-s', 'EMULATED_FUNCTION_POINTERS=1'], - 31, ['abort'], ['waka'], 186423, 29, 39, 519), # noqa + 33, ['abort'], ['waka'], 186423, 29, 39, 519), # noqa }) @no_wasm_backend() def test_binaryen_metadce_cxx_fastcomp(self, *args): @@ -7955,9 +7963,9 @@ def test_binaryen_metadce_cxx_fastcomp(self, *args): self.run_metadce_test('hello_libcxx.cpp', *args) @parameterized({ - 'O0': ([], 17, [], ['waka'], 22185, 11, 17, 57), # noqa - 'O1': (['-O1'], 15, [], ['waka'], 10415, 9, 14, 31), # noqa - 'O2': (['-O2'], 15, [], ['waka'], 10183, 9, 14, 25), # noqa + 'O0': ([], 17, [], ['waka'], 22185, 11, 19, 57), # noqa + 'O1': (['-O1'], 15, [], ['waka'], 10415, 9, 16, 31), # noqa + 'O2': (['-O2'], 15, [], ['waka'], 10183, 9, 16, 25), # noqa 'O3': (['-O3'], 5, [], [], 2353, 7, 3, 14), # noqa; in -O3, -Os and -Oz we metadce 'Os': (['-Os'], 5, [], [], 2310, 7, 3, 15), # noqa 'Oz': (['-Oz'], 5, [], [], 2272, 7, 2, 14), # noqa @@ -7969,7 +7977,7 @@ def test_binaryen_metadce_cxx_fastcomp(self, *args): # don't compare the # of functions in a main module, which changes a lot # TODO(sbc): Investivate why the number of exports is order of magnitude # larger for wasm backend. - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1605, [], [], 517336, 172, 1484, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1605, [], [], 517336, 172, 1506, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10770, 17, 13, None), # noqa }) @no_fastcomp() @@ -7989,7 +7997,7 @@ def test_binaryen_metadce_hello(self, *args): 0, [], [], 8, 0, 0, 0), # noqa; totally empty! # we don't metadce with linkable code! other modules may want stuff # don't compare the # of functions in a main module, which changes a lot - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1570, [], [], 226403, 30, 96, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1570, [], [], 226403, 30, 97, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10571, 19, 9, 21), # noqa }) @no_wasm_backend() From 1dc762cfb4a21e0dfaee2cfa689a78bb594ff95d Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Fri, 7 Jun 2019 11:04:31 -0700 Subject: [PATCH 24/62] Make embuilder build lsan --- embuilder.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/embuilder.py b/embuilder.py index 78d942c52a496..678976d8b1446 100755 --- a/embuilder.py +++ b/embuilder.py @@ -69,6 +69,7 @@ 'pthreads_stub', 'ubsan-rt-wasm', 'ubsan-minimal-rt-wasm', + 'lsan-rt-wasm', ] for debug in ['', '_debug']: @@ -336,6 +337,10 @@ def is_flag(arg): if not shared.Settings.WASM_BACKEND: continue build(C_BARE, ['libubsan_minimal_rt_wasm.a'], ['-fsanitize=undefined', '-fsanitize-minimal-runtime', '-s', 'WASM=1']) + elif what == 'lsan-rt-wasm': + if not shared.Settings.WASM_BACKEND: + continue + build(C_BARE, ['liblsan_rt_wasm.a'], ['-fsanitize=leak', '-s', 'WASM=1']) elif what == 'al': build(''' #include "AL/al.h" From fffba0a470596548ce307c05c3f9fc1f0ff85403 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Fri, 7 Jun 2019 11:13:51 -0700 Subject: [PATCH 25/62] Update ChangeLog.md --- ChangeLog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog.md b/ChangeLog.md index b6052d3f4546c..5ff265944f181 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -18,6 +18,7 @@ See docs/process.md for how version tagging works. Current Trunk ------------- +- Add support for standalone [leak sanitizer](https://clang.llvm.org/docs/LeakSanitizer.html). (#8711) v1.38.36: 06/15/2019 -------------------- From f5e53cb529764241d406acd2c4d301fe604a1457 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Fri, 7 Jun 2019 11:38:36 -0700 Subject: [PATCH 26/62] Fix LSan treating everything as leaks --- system/lib/compiler-rt/lib/lsan/lsan_common.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.cc b/system/lib/compiler-rt/lib/lsan/lsan_common.cc index 257fa7a156c6b..4f38559eb8c8c 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.cc @@ -196,7 +196,7 @@ void ScanRangeForPointers(uptr begin, uptr end, } #if SANITIZER_EMSCRIPTEN - if (cache_begin <= pp || pp < cache_end) { + if (cache_begin <= pp && pp < cache_end) { LOG_POINTERS("%p: skipping because it overlaps the cache %p-%p.\n", pp, cache_begin, cache_end); continue; From 5c18aa1a1fd1ff9db48837bc555610ea8b5519f5 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Fri, 7 Jun 2019 11:42:32 -0700 Subject: [PATCH 27/62] Add LSan tests --- tests/core/test_lsan_leaks.c | 14 ++++++++++++++ tests/core/test_lsan_no_leak.c | 10 ++++++++++ tests/runner.py | 6 ++++-- tests/test_core.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 tests/core/test_lsan_leaks.c create mode 100644 tests/core/test_lsan_no_leak.c diff --git a/tests/core/test_lsan_leaks.c b/tests/core/test_lsan_leaks.c new file mode 100644 index 0000000000000..ec5f5e5dbaaf1 --- /dev/null +++ b/tests/core/test_lsan_leaks.c @@ -0,0 +1,14 @@ +#include + +void *global_ptr; + +void f(void) { + void *local_ptr = malloc(42); +} + +int main(int argc, char **argv) { + global_ptr = malloc(1337); + global_ptr = 0; + f(); + malloc(2048); +} diff --git a/tests/core/test_lsan_no_leak.c b/tests/core/test_lsan_no_leak.c new file mode 100644 index 0000000000000..910be5a067903 --- /dev/null +++ b/tests/core/test_lsan_no_leak.c @@ -0,0 +1,10 @@ +#include + +void *global_ptr; + +int main(int argc, char **argv) { + void *local_ptr = malloc(42); + free(local_ptr); + global_ptr = malloc(1337); + free(malloc(2048)); +} diff --git a/tests/runner.py b/tests/runner.py index 5bf43db6deed8..6640065ab7cb8 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -1087,7 +1087,8 @@ def do_run(self, src, expected_output, args=[], output_nicerizer=None, no_build=False, main_file=None, additional_files=[], js_engines=None, post_build=None, basename='src.cpp', libraries=[], includes=[], force_c=False, build_ll_hook=None, - assert_returncode=None, assert_identical=False, assert_all=False): + assert_returncode=None, assert_identical=False, assert_all=False, + check_for_error=True): if self.get_setting('ASYNCIFY') == 1 and self.is_wasm_backend(): self.skipTest("wasm backend doesn't support ASYNCIFY yet") if force_c or (main_file is not None and main_file[-2:]) == '.c': @@ -1128,7 +1129,8 @@ def do_run(self, src, expected_output, args=[], output_nicerizer=None, self.assertIdentical(expected_output, js_output) else: self.assertContained(expected_output, js_output, check_all=assert_all) - self.assertNotContained('ERROR', js_output) + if check_for_error: + self.assertNotContained('ERROR', js_output) except Exception: print('(test did not pass in JS engine: %s)' % engine) raise diff --git a/tests/test_core.py b/tests/test_core.py index 93f1efe807827..a2e9ff6db8aa4 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -7751,6 +7751,40 @@ def modify_env(filename): self.do_run(open(path_from_root('tests', 'core', 'test_ubsan_full_null_ref.cpp')).read(), post_build=modify_env, assert_all=True, expected_output=expected_output) + @no_fastcomp('lsan not supported on fastcomp') + def test_lsan_leaks(self): + self.emcc_args += ['-fsanitize=leak'] + self.set_setting('ALLOW_MEMORY_GROWTH', 1) + self.do_run(open(path_from_root('tests', 'core', 'test_lsan_leaks.c')).read(), + assert_all=True, check_for_error=False, expected_output=[ + 'Direct leak of 2048 byte(s) in 1 object(s) allocated from', + 'Direct leak of 1337 byte(s) in 1 object(s) allocated from', + 'Direct leak of 42 byte(s) in 1 object(s) allocated from', + ]) + + @no_fastcomp('lsan not supported on fastcomp') + def test_lsan_stack_trace(self): + self.emcc_args += ['-fsanitize=leak', '-g4'] + self.set_setting('ALLOW_MEMORY_GROWTH', 1) + self.do_run(open(path_from_root('tests', 'core', 'test_lsan_leaks.c')).read(), + assert_all=True, check_for_error=False, expected_output=[ + 'Direct leak of 2048 byte(s) in 1 object(s) allocated from', + 'Direct leak of 1337 byte(s) in 1 object(s) allocated from', + 'Direct leak of 42 byte(s) in 1 object(s) allocated from', + 'in main', + '/src.cpp:6:21', + '/src.cpp:10:16', + '/src.cpp:12:3', + '/src.cpp:13:3', + ]) + + @no_fastcomp('lsan not supported on fastcomp') + def test_lsan_no_leak(self): + self.emcc_args += ['-fsanitize=leak'] + self.set_setting('ALLOW_MEMORY_GROWTH', 1) + self.do_run(open(path_from_root('tests', 'core', 'test_lsan_no_leak.c')).read(), + expected_output=['']) + # Generate tests for everything def make_run(name, emcc_args, settings=None, env=None): From 641793c8d3209286dab88d017caa7ff7a15d8699 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Fri, 7 Jun 2019 11:52:28 -0700 Subject: [PATCH 28/62] Do not use source map beyond function boundaries --- src/source_map_support.js | 7 ++++++- src/wasm_offset_converter.js | 27 +++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/source_map_support.js b/src/source_map_support.js index e7b25ff329c98..a98e658d41e72 100644 --- a/src/source_map_support.js +++ b/src/source_map_support.js @@ -55,7 +55,12 @@ function WasmSourceMap(sourceMap) { } WasmSourceMap.prototype.lookup = function (offset) { - var info = this.mapping[this.normalizeOffset(offset)]; + var normalized = this.normalizeOffset(offset); +#if 'emscripten_generate_pc' in addedLibraryItems + if (!wasmOffsetConverter.isSameFunc(offset, normalized)) + return null; +#endif + var info = this.mapping[normalized]; if (!info) return null; return { diff --git a/src/wasm_offset_converter.js b/src/wasm_offset_converter.js index 3704b5621f87d..dbdf8df496de4 100644 --- a/src/wasm_offset_converter.js +++ b/src/wasm_offset_converter.js @@ -9,8 +9,9 @@ function WasmOffsetConverter(wasmBytes) { // WASM stack trace format. // // v8 bug: https://crbug.com/v8/9172 - // We may be able to remove this code once the fix makes its way into all - // commonly used versions of node. + // + // This code is also used to check if the candidate source map offset is + // actually part of the same function as the offset we are looking for. // current byte offset into the WASM binary, as we parse it // the first section starts at offset 8 @@ -21,6 +22,7 @@ function WasmOffsetConverter(wasmBytes) { // map from function index to byte offset in WASM binary this.map = {}; + this.func_starts = []; function unsignedLEB128() { // consumes an unsigned LEB128 integer starting at `offset`. @@ -82,6 +84,7 @@ function WasmOffsetConverter(wasmBytes) { while (count-- > 0) { var size = unsignedLEB128(); this.map[funcidx++] = offset; + this.func_starts.push(offset); offset += size; } return; @@ -93,3 +96,23 @@ function WasmOffsetConverter(wasmBytes) { WasmOffsetConverter.prototype.convert = function (funcidx, offset) { return this.map[funcidx] + offset; } + +WasmOffsetConverter.prototype.isSameFunc = function (offset1, offset2) { + var array = this.func_starts; + function bisect(offset) { + var lo = 0; + var hi = array.length; + var mid; + + while (lo < hi) { + mid = Math.floor((lo + hi) / 2); + if (array[mid] > offset) { + hi = mid; + } else { + lo = mid + 1; + } + } + return lo; + } + return bisect(offset1) == bisect(offset2); +} From 250dd2f3eede474e121105fd74d2e315af93ea7a Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Fri, 7 Jun 2019 13:04:25 -0700 Subject: [PATCH 29/62] LSan is just a link time thing, no need to optimize source in test --- tests/{core => other}/test_lsan_leaks.c | 0 tests/{core => other}/test_lsan_no_leak.c | 0 tests/test_core.py | 34 ---------------- tests/test_other.py | 47 +++++++++++++++++++++++ 4 files changed, 47 insertions(+), 34 deletions(-) rename tests/{core => other}/test_lsan_leaks.c (100%) rename tests/{core => other}/test_lsan_no_leak.c (100%) diff --git a/tests/core/test_lsan_leaks.c b/tests/other/test_lsan_leaks.c similarity index 100% rename from tests/core/test_lsan_leaks.c rename to tests/other/test_lsan_leaks.c diff --git a/tests/core/test_lsan_no_leak.c b/tests/other/test_lsan_no_leak.c similarity index 100% rename from tests/core/test_lsan_no_leak.c rename to tests/other/test_lsan_no_leak.c diff --git a/tests/test_core.py b/tests/test_core.py index a2e9ff6db8aa4..93f1efe807827 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -7751,40 +7751,6 @@ def modify_env(filename): self.do_run(open(path_from_root('tests', 'core', 'test_ubsan_full_null_ref.cpp')).read(), post_build=modify_env, assert_all=True, expected_output=expected_output) - @no_fastcomp('lsan not supported on fastcomp') - def test_lsan_leaks(self): - self.emcc_args += ['-fsanitize=leak'] - self.set_setting('ALLOW_MEMORY_GROWTH', 1) - self.do_run(open(path_from_root('tests', 'core', 'test_lsan_leaks.c')).read(), - assert_all=True, check_for_error=False, expected_output=[ - 'Direct leak of 2048 byte(s) in 1 object(s) allocated from', - 'Direct leak of 1337 byte(s) in 1 object(s) allocated from', - 'Direct leak of 42 byte(s) in 1 object(s) allocated from', - ]) - - @no_fastcomp('lsan not supported on fastcomp') - def test_lsan_stack_trace(self): - self.emcc_args += ['-fsanitize=leak', '-g4'] - self.set_setting('ALLOW_MEMORY_GROWTH', 1) - self.do_run(open(path_from_root('tests', 'core', 'test_lsan_leaks.c')).read(), - assert_all=True, check_for_error=False, expected_output=[ - 'Direct leak of 2048 byte(s) in 1 object(s) allocated from', - 'Direct leak of 1337 byte(s) in 1 object(s) allocated from', - 'Direct leak of 42 byte(s) in 1 object(s) allocated from', - 'in main', - '/src.cpp:6:21', - '/src.cpp:10:16', - '/src.cpp:12:3', - '/src.cpp:13:3', - ]) - - @no_fastcomp('lsan not supported on fastcomp') - def test_lsan_no_leak(self): - self.emcc_args += ['-fsanitize=leak'] - self.set_setting('ALLOW_MEMORY_GROWTH', 1) - self.do_run(open(path_from_root('tests', 'core', 'test_lsan_no_leak.c')).read(), - expected_output=['']) - # Generate tests for everything def make_run(name, emcc_args, settings=None, env=None): diff --git a/tests/test_other.py b/tests/test_other.py index 1b8dc05b62716..61a592d2e3308 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7,6 +7,7 @@ # noqa: E241 from __future__ import print_function +from functools import wraps import difflib import filecmp import glob @@ -64,6 +65,7 @@ def uses_canonical_tmp(func): This decorator takes care of cleaning the directory after the test to satisfy the leak detector. """ + @wraps(func) def decorated(self): # Before running the test completely remove the canonical_tmp if os.path.exists(self.canonical_temp_dir): @@ -134,6 +136,19 @@ def do_other_test(self, dirname, emcc_args=[], run_args=[]): seen = run_js('a.out.js', args=run_args, stderr=PIPE, full_output=True) + '\n' self.assertContained(expected, seen) + def do_smart_test(self, source, literals=[], regexes=[], emcc_args=[], run_args=[], assert_returncode=0): + shutil.copyfile(source, 'test.cpp') + run_process([PYTHON, EMCC, 'test.cpp'] + emcc_args) + seen = run_js('a.out.js', args=run_args, stderr=PIPE, full_output=True, + assert_returncode=assert_returncode) + '\n' + + for literal in literals: + self.assertContained([literal], seen) + + for regex in regexes: + if not re.search(regex, seen): + self.fail('Expected regex "%s" to match on:\n%s' % (regex, seen)) + def expect_fail(self, cmd, **args): """Run a subprocess and assert that it returns non-zero. @@ -9280,3 +9295,35 @@ def test(code): def test_malloc_none(self): stderr = self.expect_fail([PYTHON, EMCC, path_from_root('tests', 'malloc_none.c'), '-s', 'MALLOC=none']) self.assertContained('undefined symbol: malloc', stderr) + + @no_fastcomp('lsan not supported on fastcomp') + def test_lsan_leaks(self): + self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_leaks.c'), + emcc_args=['-fsanitize=leak', '-s', 'ALLOW_MEMORY_GROWTH=1'], + assert_returncode=None, literals=[ + 'Direct leak of 2048 byte(s) in 1 object(s) allocated from', + 'Direct leak of 1337 byte(s) in 1 object(s) allocated from', + 'Direct leak of 42 byte(s) in 1 object(s) allocated from', + ]) + + @no_fastcomp('lsan not supported on fastcomp') + def test_lsan_stack_trace(self): + self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_leaks.c'), + emcc_args=['-fsanitize=leak', '-s', 'ALLOW_MEMORY_GROWTH=1', '-g4'], + assert_returncode=None, literals=[ + 'Direct leak of 2048 byte(s) in 1 object(s) allocated from', + 'Direct leak of 1337 byte(s) in 1 object(s) allocated from', + 'Direct leak of 42 byte(s) in 1 object(s) allocated from', + ], regexes=[ + r'(?m)in malloc.*wasm-function', + r'(?m)in f\(\) /.*/test\.cpp:6:21$', + r'(?m)in main /.*/test\.cpp:10:16$', + r'(?m)in main /.*/test\.cpp:12:3$', + r'(?m)in main /.*/test\.cpp:13:3$', + ]) + + @no_fastcomp('lsan not supported on fastcomp') + def test_lsan_no_leak(self): + self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_no_leak.c'), + emcc_args=['-fsanitize=leak', '-s', 'ALLOW_MEMORY_GROWTH=1'], + regexes=[r'^\s*$']) From f1105d9a43ebe1c3880c4a01a38df678c1926cef Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Fri, 7 Jun 2019 17:15:03 -0700 Subject: [PATCH 30/62] Add C++ operator new tests for LSan --- tests/other/test_lsan_leaks.cpp | 12 +++++++ tests/other/test_lsan_no_leak.cpp | 10 ++++++ tests/test_other.py | 57 ++++++++++++++++++++----------- 3 files changed, 60 insertions(+), 19 deletions(-) create mode 100644 tests/other/test_lsan_leaks.cpp create mode 100644 tests/other/test_lsan_no_leak.cpp diff --git a/tests/other/test_lsan_leaks.cpp b/tests/other/test_lsan_leaks.cpp new file mode 100644 index 0000000000000..03e3172e6dbda --- /dev/null +++ b/tests/other/test_lsan_leaks.cpp @@ -0,0 +1,12 @@ +void *global_ptr; + +void f(void) { + void *local_ptr = new short[21]; +} + +int main(int argc, char **argv) { + global_ptr = new char[1337]; + global_ptr = 0; + f(); + new long long[256]; +} diff --git a/tests/other/test_lsan_no_leak.cpp b/tests/other/test_lsan_no_leak.cpp new file mode 100644 index 0000000000000..5e40cc13a7087 --- /dev/null +++ b/tests/other/test_lsan_no_leak.cpp @@ -0,0 +1,10 @@ +#include + +void *global_ptr; + +int main(int argc, char **argv) { + void *local_ptr = new short[21]; + free(local_ptr); + global_ptr = new char[1337]; + delete [] new long long[256]; +} diff --git a/tests/test_other.py b/tests/test_other.py index 61a592d2e3308..08480cbcf24d2 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -136,9 +136,10 @@ def do_other_test(self, dirname, emcc_args=[], run_args=[]): seen = run_js('a.out.js', args=run_args, stderr=PIPE, full_output=True) + '\n' self.assertContained(expected, seen) - def do_smart_test(self, source, literals=[], regexes=[], emcc_args=[], run_args=[], assert_returncode=0): - shutil.copyfile(source, 'test.cpp') - run_process([PYTHON, EMCC, 'test.cpp'] + emcc_args) + def do_smart_test(self, source, name='test.cpp', literals=[], regexes=[], + emcc_args=[], run_args=[], assert_returncode=0): + shutil.copyfile(source, name) + run_process([PYTHON, EMCC, name] + emcc_args) seen = run_js('a.out.js', args=run_args, stderr=PIPE, full_output=True, assert_returncode=assert_returncode) + '\n' @@ -9296,34 +9297,52 @@ def test_malloc_none(self): stderr = self.expect_fail([PYTHON, EMCC, path_from_root('tests', 'malloc_none.c'), '-s', 'MALLOC=none']) self.assertContained('undefined symbol: malloc', stderr) + @parameterized({ + 'c': ['c'], + 'cpp': ['cpp'], + }) @no_fastcomp('lsan not supported on fastcomp') - def test_lsan_leaks(self): - self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_leaks.c'), + def test_lsan_leaks(self, ext): + self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_leaks.' + ext), emcc_args=['-fsanitize=leak', '-s', 'ALLOW_MEMORY_GROWTH=1'], - assert_returncode=None, literals=[ + name='test.' + ext, assert_returncode=None, literals=[ 'Direct leak of 2048 byte(s) in 1 object(s) allocated from', 'Direct leak of 1337 byte(s) in 1 object(s) allocated from', 'Direct leak of 42 byte(s) in 1 object(s) allocated from', ]) + @parameterized({ + 'c': ['c', [ + r'in malloc.*wasm-function', + r'(?m)in f /.*/test\.c:6:21$', + r'(?m)in main /.*/test\.c:10:16$', + r'(?m)in main /.*/test\.c:12:3$', + r'(?m)in main /.*/test\.c:13:3$', + ]], + 'cpp': ['cpp', [ + r'in operator new\[\]\(unsigned long\).*wasm-function', + r'(?m)in f\(\) /.*/test\.cpp:4:21$', + r'(?m)in main /.*/test\.cpp:8:16$', + r'(?m)in main /.*/test\.cpp:10:3$', + r'(?m)in main /.*/test\.cpp:11:3$', + ]], + }) @no_fastcomp('lsan not supported on fastcomp') - def test_lsan_stack_trace(self): - self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_leaks.c'), + def test_lsan_stack_trace(self, ext, regexes): + self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_leaks.' + ext), emcc_args=['-fsanitize=leak', '-s', 'ALLOW_MEMORY_GROWTH=1', '-g4'], - assert_returncode=None, literals=[ + name='test.' + ext, assert_returncode=None, literals=[ 'Direct leak of 2048 byte(s) in 1 object(s) allocated from', 'Direct leak of 1337 byte(s) in 1 object(s) allocated from', 'Direct leak of 42 byte(s) in 1 object(s) allocated from', - ], regexes=[ - r'(?m)in malloc.*wasm-function', - r'(?m)in f\(\) /.*/test\.cpp:6:21$', - r'(?m)in main /.*/test\.cpp:10:16$', - r'(?m)in main /.*/test\.cpp:12:3$', - r'(?m)in main /.*/test\.cpp:13:3$', - ]) + ], regexes=regexes) + @parameterized({ + 'c': ['c'], + 'cpp': ['cpp'], + }) @no_fastcomp('lsan not supported on fastcomp') - def test_lsan_no_leak(self): - self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_no_leak.c'), + def test_lsan_no_leak(self, ext): + self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_no_leak.' + ext), emcc_args=['-fsanitize=leak', '-s', 'ALLOW_MEMORY_GROWTH=1'], - regexes=[r'^\s*$']) + name='test.' + ext, regexes=[r'^\s*$']) From 7d8817c37ae65f134b4e7316d261409284d062a2 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 10 Jun 2019 14:12:31 -0700 Subject: [PATCH 31/62] LSan does not yet support threads --- emcc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/emcc.py b/emcc.py index 87e426944dac8..d223714899be0 100755 --- a/emcc.py +++ b/emcc.py @@ -1486,6 +1486,9 @@ def check(input_file): shared.Settings.USE_LSAN = 1 shared.Settings.EXIT_RUNTIME = 1 + if shared.Settings.USE_PTHREADS: + exit_with_error('LSan currently does not support threads') + if sanitize and '-g4' in args: shared.Settings.LOAD_SOURCE_MAP = 1 From c6cd0ee565c0807dd308e9b408fd46fe8c725962 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 10 Jun 2019 15:28:38 -0700 Subject: [PATCH 32/62] Add lsan unit test mode --- tests/test_core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 93f1efe807827..801bb3f47c02e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -7845,5 +7845,7 @@ def setUp(self): asmi = make_run('asmi', emcc_args=[], settings={'ASM_JS': 2, 'EMTERPRETIFY': 1, 'WASM': 0}) asm2i = make_run('asm2i', emcc_args=['-O2'], settings={'EMTERPRETIFY': 1, 'WASM': 0}) +lsan = make_run('lsan', emcc_args=['-fsanitize=leak'], settings={'ALLOW_MEMORY_GROWTH': 1}) + # TestCoreBase is just a shape for the specific subclasses, we don't test it itself del TestCoreBase # noqa From cf037138f6e8672d385a74881314f7f7f9445256 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 17 Jun 2019 17:21:52 -0700 Subject: [PATCH 33/62] fix metadce tests --- tests/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index 84c6844987e2b..10a6259bc97bd 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8013,7 +8013,7 @@ def test_binaryen_metadce_hello(self, *args): 0, [], [], 8, 0, 0, 0), # noqa; totally empty! # we don't metadce with linkable code! other modules may want stuff # don't compare the # of functions in a main module, which changes a lot - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1570, [], [], 226403, 30, 97, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1570, [], [], 226403, 30, 96, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10571, 19, 9, 21), # noqa }) @no_wasm_backend() From e80236d84039d1adc2b0fdfd49ccf546e7d77d56 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 17 Jun 2019 17:28:46 -0700 Subject: [PATCH 34/62] Disable multithread builds for LSan --- tools/system_libs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index dff019115fd7d..abffafcc8ebcc 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -991,11 +991,16 @@ class libubsan_rt_wasm(SanitizerLibrary): src_dir = ['system', 'lib', 'compiler-rt', 'lib', 'ubsan'] -class liblsan_rt_wasm(SanitizerLibrary): +# TODO: once thread local storage is implemented, make this inherit from SanitizerLibrary +# and clean up the duplicate code. +class liblsan_rt_wasm(CompilerRTWasmLibrary): name = 'liblsan_rt_wasm' + depends = ['libsanitizer_common_rt_wasm'] js_depends = ['__data_end', '__heap_base'] - cflags = ['-DUBSAN_CAN_USE_CXXABI'] + includes = [['system', 'lib', 'compiler-rt', 'lib']] + cflags = ['-std=c++11'] + src_glob = '*.cc' src_dir = ['system', 'lib', 'compiler-rt', 'lib', 'lsan'] From 664fd841be63a77f825ad27e96eade617ac44d58 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 17 Jun 2019 18:22:44 -0700 Subject: [PATCH 35/62] Fix CI tests --- tests/test_other.py | 4 ++-- tools/system_libs.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index 10a6259bc97bd..81c1c5a1b4d25 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -9313,14 +9313,14 @@ def test_lsan_leaks(self, ext): @parameterized({ 'c': ['c', [ - r'in malloc.*wasm-function', + r'in malloc.*a\.out\.wasm\+0x', r'(?m)in f /.*/test\.c:6:21$', r'(?m)in main /.*/test\.c:10:16$', r'(?m)in main /.*/test\.c:12:3$', r'(?m)in main /.*/test\.c:13:3$', ]], 'cpp': ['cpp', [ - r'in operator new\[\]\(unsigned long\).*wasm-function', + r'in operator new\[\]\(unsigned long\).*a\.out\.wasm\+0x', r'(?m)in f\(\) /.*/test\.cpp:4:21$', r'(?m)in main /.*/test\.cpp:8:16$', r'(?m)in main /.*/test\.cpp:10:3$', diff --git a/tools/system_libs.py b/tools/system_libs.py index abffafcc8ebcc..4a28b300acf1f 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -178,6 +178,9 @@ class Library(object): # dependencies of this library working. js_depends = [] + # Set to true to prevent EMCC_FORCE_STDLIBS from linking this library. + never_force = False + # The C compile executable to use. You can override this to shared.EMXX for C++. emcc = shared.EMCC @@ -960,6 +963,7 @@ def get_files(self): class libubsan_minimal_rt_wasm(CompilerRTWasmLibrary, MTLibrary): name = 'libubsan_minimal_rt_wasm' + never_force = True src_dir = ['system', 'lib', 'compiler-rt', 'lib', 'ubsan_minimal'] src_files = ['ubsan_minimal_handlers.cpp'] @@ -969,6 +973,7 @@ class libsanitizer_common_rt_wasm(CompilerRTWasmLibrary, MTLibrary): name = 'libsanitizer_common_rt_wasm' depends = ['libc++abi'] js_depends = ['memalign', 'emscripten_builtin_memalign'] + never_force = True cflags = ['-std=c++11'] src_dir = ['system', 'lib', 'compiler-rt', 'lib', 'sanitizer_common'] @@ -978,6 +983,7 @@ class libsanitizer_common_rt_wasm(CompilerRTWasmLibrary, MTLibrary): class SanitizerLibrary(CompilerRTWasmLibrary, MTLibrary): depends = ['libsanitizer_common_rt_wasm'] + never_force = True includes = [['system', 'lib', 'compiler-rt', 'lib']] cflags = ['-std=c++11'] @@ -997,6 +1003,7 @@ class liblsan_rt_wasm(CompilerRTWasmLibrary): name = 'liblsan_rt_wasm' depends = ['libsanitizer_common_rt_wasm'] js_depends = ['__data_end', '__heap_base'] + never_force = True includes = [['system', 'lib', 'compiler-rt', 'lib']] cflags = ['-std=c++11'] @@ -1086,7 +1093,7 @@ class Dummy(object): # You can provide 1 to include everything, or a comma-separated list with the ones you want force = os.environ.get('EMCC_FORCE_STDLIBS') if force == '1': - force = ','.join(system_libs_map.keys()) + force = ','.join(key for key, value in system_libs_map.items() if not value.never_force) force_include = set((force.split(',') if force else []) + forced) if force_include: logger.debug('forcing stdlibs: ' + str(force_include)) From 1a926d10d1acdf90b0d8d8e92cc53f9375ef05d6 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Tue, 18 Jun 2019 14:48:43 -0700 Subject: [PATCH 36/62] Fix comments as requested --- system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc | 2 +- system/lib/dlmalloc.c | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc b/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc index c89a916ae513d..4137256881bf3 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc @@ -91,7 +91,7 @@ LoadedModule *GetLinker() { return linker; } void ProcessPlatformSpecificAllocations(Frontier *frontier) {} // While calling Die() here is undefined behavior and can potentially -// cause race conditions, it isn't possible to intercept exit on linux, +// cause race conditions, it isn't possible to intercept exit on Emscripten, // so we have no choice but to call Die() from the atexit handler. void HandleLeaks() { if (common_flags()->exitcode) Die(); diff --git a/system/lib/dlmalloc.c b/system/lib/dlmalloc.c index a34c5b9adf89f..b0be79d16cf69 100644 --- a/system/lib/dlmalloc.c +++ b/system/lib/dlmalloc.c @@ -852,8 +852,6 @@ extern "C" { #ifndef USE_DL_PREFIX // XXX Emscripten XXX #if defined(__EMSCRIPTEN__) && !defined(__asmjs__) -// XXX fastcomp HACK: in fastcomp, weak aliases loses to JavaScript versions of the function. -// Therefore, we must define real functions. Remove this hack when fastcomp is removed. void* malloc(size_t) __attribute__((weak, alias("dlmalloc"))); void free(void*) __attribute__((weak, alias("dlfree"))); void* calloc(size_t, size_t) __attribute__((weak, alias("dlcalloc"))); @@ -881,6 +879,10 @@ void** independent_calloc(size_t, size_t, void**) __attribute__((weak, alias("dl void** independent_comalloc(size_t, size_t*, void**) __attribute__((weak, alias("dlindependent_comalloc"))); size_t bulk_free(void**, size_t n_elements) __attribute__((weak, alias("dlbulk_free"))); #else +// XXX fastcomp HACK: in fastcomp, weak aliases loses to JavaScript versions of the function. +// Therefore, we must define real functions. We use macros to remove the dl prefix, but this +// forces the user to override all or none of the functions below. +// Remove the else block once fastcomp is removed. #define dlcalloc calloc #define dlfree free #define dlmalloc malloc @@ -6065,6 +6067,8 @@ int mspace_mallopt(int param_number, int value) { // This allows an easy mechanism for hooking into memory allocation. #if defined(__EMSCRIPTEN__) && !ONLY_MSPACES #ifdef __asmjs__ +// XXX This is to support the fastcomp hack above where we remove the dl prefix. +// TODO: Remove this branch when fastcomp is removed. extern __typeof(malloc) emscripten_builtin_malloc __attribute__((alias("malloc"))); extern __typeof(free) emscripten_builtin_free __attribute__((alias("free"))); extern __typeof(memalign) emscripten_builtin_memalign __attribute__((alias("memalign"))); From c1a473ce6fdc9226fdd7692a27407dd5788a306a Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Tue, 18 Jun 2019 14:52:59 -0700 Subject: [PATCH 37/62] flake8 --- tests/test_other.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index 5e41acc0a9d7d..d300a2d25e942 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -9377,4 +9377,3 @@ def test_build_error_color(self): def test_llvm_includes(self): self.build('#include ', self.get_dir(), 'atomics.c') - From 980b7ddf1430b9879a3333b9195cd3ba4d862efa Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 19 Jun 2019 18:02:11 -0700 Subject: [PATCH 38/62] Preemptively address review comments --- src/source_map_support.js | 6 ++++-- .../lib/sanitizer_common/sanitizer_emscripten.cc | 2 +- system/lib/dlmalloc.c | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/source_map_support.js b/src/source_map_support.js index a98e658d41e72..b5eea48d167b2 100644 --- a/src/source_map_support.js +++ b/src/source_map_support.js @@ -57,12 +57,14 @@ function WasmSourceMap(sourceMap) { WasmSourceMap.prototype.lookup = function (offset) { var normalized = this.normalizeOffset(offset); #if 'emscripten_generate_pc' in addedLibraryItems - if (!wasmOffsetConverter.isSameFunc(offset, normalized)) + if (!wasmOffsetConverter.isSameFunc(offset, normalized)) { return null; + } #endif var info = this.mapping[normalized]; - if (!info) + if (!info) { return null; + } return { source: this.sources[info.source], line: info.line, diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc index c58b1bbbadf8a..f9774724f76cd 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc @@ -91,7 +91,7 @@ uptr GetTlsSize() { return 0; } -void InitTlsSize() { } +void InitTlsSize() {} void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, uptr *tls_addr, uptr *tls_size) { diff --git a/system/lib/dlmalloc.c b/system/lib/dlmalloc.c index b0be79d16cf69..c0a2992c512d4 100644 --- a/system/lib/dlmalloc.c +++ b/system/lib/dlmalloc.c @@ -1,7 +1,7 @@ /* XXX Emscripten XXX */ #if __EMSCRIPTEN__ -#define DLMALLOC_EXPORT __attribute__((weak)) +#define DLMALLOC_EXPORT __attribute__((__weak__)) /* mmap uses malloc, so malloc can't use mmap */ #define HAVE_MMAP 0 /* we can only grow the heap up anyhow, so don't try to trim */ From f0352e24ba721d62e371ddd2e2d6639e2bda9174 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Thu, 20 Jun 2019 15:22:51 -0700 Subject: [PATCH 39/62] Load LSan options from Module with emscripten_builtin_malloc/free This removes the nasty hack with getenv --- emcc.py | 2 +- src/deps_info.json | 4 ++-- src/library.js | 9 +-------- system/lib/compiler-rt/lib/lsan/lsan.cc | 18 ++++++++++++++++++ tools/system_libs.py | 2 +- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/emcc.py b/emcc.py index d223714899be0..0e989d20859b0 100755 --- a/emcc.py +++ b/emcc.py @@ -1203,7 +1203,7 @@ def check(input_file): if shared.Settings.WASM_BACKEND: # setjmp/longjmp and exception handling JS code depends on this so we # include it by default. Should be eliminated by meta-DCE if unused. - shared.Settings.EXPORTED_FUNCTIONS += ['_setThrew', '_emscripten_builtin_malloc', '_emscripten_builtin_free'] + shared.Settings.EXPORTED_FUNCTIONS += ['_setThrew'] if shared.Settings.RELOCATABLE and not shared.Settings.DYNAMIC_EXECUTION: exit_with_error('cannot have both DYNAMIC_EXECUTION=0 and RELOCATABLE enabled at the same time, since RELOCATABLE needs to eval()') diff --git a/src/deps_info.json b/src/deps_info.json index 5cf2ca0c9aecc..6b8c3f301e4ba 100644 --- a/src/deps_info.json +++ b/src/deps_info.json @@ -2,7 +2,7 @@ "arc4random": ["rand"], "freopen": ["free"], "munmap": ["emscripten_builtin_free", "free"], - "getenv": ["emscripten_builtin_malloc", "emscripten_builtin_free", "malloc", "free", "_get_environ"], + "getenv": ["malloc", "free", "_get_environ"], "clearenv": ["_get_environ"], "setenv": ["_get_environ"], "unsetenv": ["_get_environ"], @@ -10,7 +10,7 @@ "environ": ["_get_environ"], "fmtmsg": ["_get_environ"], "get_current_dir_name": ["_get_environ"], - "SDL_getenv": ["emscripten_builtin_malloc", "emscripten_builtin_free", "malloc", "free", "_get_environ"], + "SDL_getenv": ["malloc", "free", "_get_environ"], "SDL_putenv": ["_get_environ"], "dlerror": ["malloc", "free"], "readdir": ["malloc"], diff --git a/src/library.js b/src/library.js index ae5391b872643..95f3ea82d82da 100644 --- a/src/library.js +++ b/src/library.js @@ -817,13 +817,11 @@ LibraryManager.library = { var ptr = dynamicAlloc(bytes + 8); return (ptr+8) & 0xFFFFFFF8; }, - emscripten_builtin_malloc: 'malloc', free: function() { #if ASSERTIONS == 2 warnOnce('using stub free (reference it from C to have the real one included)'); #endif }, - emscripten_builtin_free: 'free', #endif abs: 'Math_abs', @@ -923,7 +921,7 @@ LibraryManager.library = { {{{ makeSetValue('envPtr', 'strings.length * ptrSize', '0', 'i8*') }}}; }, $ENV: {}, - getenv__deps: ['$ENV', 'emscripten_builtin_malloc', 'emscripten_builtin_free'], + getenv__deps: ['$ENV'], getenv__proxy: 'sync', getenv__sig: 'ii', getenv: function(name) { @@ -933,13 +931,8 @@ LibraryManager.library = { name = UTF8ToString(name); if (!ENV.hasOwnProperty(name)) return 0; -#if WASM_BACKEND - if (_getenv.ret) _emscripten_builtin_free(_getenv.ret); - _getenv.ret = allocateUTF8(ENV[name], _emscripten_builtin_malloc); -#else if (_getenv.ret) _free(_getenv.ret); _getenv.ret = allocateUTF8(ENV[name]); -#endif return _getenv.ret; }, // Alias for sanitizers which intercept getenv. diff --git a/system/lib/compiler-rt/lib/lsan/lsan.cc b/system/lib/compiler-rt/lib/lsan/lsan.cc index 8e9f3cf1904da..021e3c9f36035 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan.cc @@ -21,6 +21,11 @@ #include "lsan_common.h" #include "lsan_thread.h" +#if SANITIZER_EMSCRIPTEN +extern "C" void emscripten_builtin_free(void *); +#include +#endif + bool lsan_inited; bool lsan_init_is_running; @@ -41,7 +46,11 @@ static void InitializeFlags() { { CommonFlags cf; cf.CopyFrom(*common_flags()); +#if !SANITIZER_EMSCRIPTEN + // getenv on emscripten uses malloc, which we can't when using LSan. + // You can't run external symbolizers anyway. cf.external_symbolizer_path = GetEnv("LSAN_SYMBOLIZER_PATH"); +#endif cf.malloc_context_size = 30; cf.intercept_tls_get_addr = true; cf.detect_leaks = true; @@ -59,7 +68,16 @@ static void InitializeFlags() { // Override from user-specified string. const char *lsan_default_options = MaybeCallLsanDefaultOptions(); parser.ParseString(lsan_default_options); +#if SANITIZER_EMSCRIPTEN + char *options = (char*) EM_ASM_INT({ + return allocateUTF8(Module['LSAN_OPTIONS'] || 0, + _emscripten_builtin_malloc); + }); + parser.ParseString(options); + emscripten_builtin_free(options); +#else parser.ParseString(GetEnv("LSAN_OPTIONS")); +#endif // SANITIZER_EMSCRIPTEN SetVerbosity(common_flags()->verbosity); diff --git a/tools/system_libs.py b/tools/system_libs.py index fe18ced2800ed..699b0ac590d9a 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1003,7 +1003,7 @@ class libubsan_rt_wasm(SanitizerLibrary): class liblsan_rt_wasm(CompilerRTWasmLibrary): name = 'liblsan_rt_wasm' depends = ['libsanitizer_common_rt_wasm'] - js_depends = ['__data_end', '__heap_base'] + js_depends = ['__data_end', '__heap_base', 'emscripten_builtin_malloc', 'emscripten_builtin_free'] never_force = True includes = [['system', 'lib', 'compiler-rt', 'lib']] From 6233f4257d54c28a4901cf24261fc6c88c187469 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Thu, 20 Jun 2019 15:26:39 -0700 Subject: [PATCH 40/62] Make UBSan load options from Module for consistency --- .../lib/compiler-rt/lib/ubsan/ubsan_flags.cc | 22 ++++++++++++++++++- tools/system_libs.py | 1 + 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_flags.cc b/system/lib/compiler-rt/lib/ubsan/ubsan_flags.cc index 7b6784b8278f6..ab9b966c001f0 100644 --- a/system/lib/compiler-rt/lib/ubsan/ubsan_flags.cc +++ b/system/lib/compiler-rt/lib/ubsan/ubsan_flags.cc @@ -20,6 +20,11 @@ #include +#if SANITIZER_EMSCRIPTEN +extern "C" void emscripten_builtin_free(void *); +#include +#endif + namespace __ubsan { const char *MaybeCallUbsanDefaultOptions() { @@ -56,7 +61,11 @@ void InitializeFlags() { CommonFlags cf; cf.CopyFrom(*common_flags()); cf.print_summary = false; +#if !SANITIZER_EMSCRIPTEN + // getenv on emscripten uses malloc, which we can't when using some sanitizers. + // You can't run external symbolizers anyway. cf.external_symbolizer_path = GetFlag("UBSAN_SYMBOLIZER_PATH"); +#endif OverrideCommonFlags(cf); } @@ -69,8 +78,19 @@ void InitializeFlags() { // Override from user-specified string. parser.ParseString(MaybeCallUbsanDefaultOptions()); + // Override from environment variable. - parser.ParseString(GetFlag("UBSAN_OPTIONS")); +#if SANITIZER_EMSCRIPTEN + char *options = (char*) EM_ASM_INT({ + return allocateUTF8(Module['UBSAN_OPTIONS'] || 0, + _emscripten_builtin_malloc); + }); + parser.ParseString(options); + emscripten_builtin_free(options); +#else + parser.ParseString(GetEnv("UBSAN_OPTIONS")); +#endif // SANITIZER_EMSCRIPTEN + InitializeCommonFlags(); if (Verbosity()) ReportUnrecognizedFlags(); diff --git a/tools/system_libs.py b/tools/system_libs.py index 699b0ac590d9a..833233549fa3b 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -993,6 +993,7 @@ class SanitizerLibrary(CompilerRTWasmLibrary, MTLibrary): class libubsan_rt_wasm(SanitizerLibrary): name = 'libubsan_rt_wasm' + js_depends = ['emscripten_builtin_malloc', 'emscripten_builtin_free'] cflags = ['-DUBSAN_CAN_USE_CXXABI'] src_dir = ['system', 'lib', 'compiler-rt', 'lib', 'ubsan'] From 415e18aaf10fbc0ddba97023d0a8d5379416fc79 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 24 Jun 2019 11:45:48 -0700 Subject: [PATCH 41/62] Fix UBSan stack trace tests --- tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index 9a7e625f70aec..dbd033e3b669b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -7774,7 +7774,7 @@ def test_ubsan_full_stack_trace(self, g_flag, expected_output): def modify_env(filename): with open(filename) as f: contents = f.read() - contents = re.sub('(?<=ENV=){}', "{'UBSAN_OPTIONS': 'print_stacktrace=1'}", contents) + contents = 'Module = {UBSAN_OPTIONS: "print_stacktrace=1"}' + contents with open(filename, 'w') as f: f.write(contents) From d1e68ea882286d4d11c7be08db76416381723f88 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 24 Jun 2019 11:50:27 -0700 Subject: [PATCH 42/62] Fix metadce tests --- .../metadce/hello_libcxx_fastcomp_O2.sent | 2 -- ...astcomp_O2_EMULATED_FUNCTION_POINTERS.sent | 2 -- .../metadce/hello_world_O3_MAIN_MODULE.sent | 1 + .../hello_world_fastcomp_O3_MAIN_MODULE.sent | 2 -- tests/test_other.py | 22 +++++++++---------- 5 files changed, 12 insertions(+), 17 deletions(-) diff --git a/tests/other/metadce/hello_libcxx_fastcomp_O2.sent b/tests/other/metadce/hello_libcxx_fastcomp_O2.sent index 0f041331b5393..2de6935c7b52e 100644 --- a/tests/other/metadce/hello_libcxx_fastcomp_O2.sent +++ b/tests/other/metadce/hello_libcxx_fastcomp_O2.sent @@ -15,8 +15,6 @@ __addDays __arraySum __isLeapYear _abort -_emscripten_builtin_free -_emscripten_builtin_malloc _emscripten_get_heap_size _emscripten_memcpy_big _emscripten_resize_heap diff --git a/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent b/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent index 0f041331b5393..2de6935c7b52e 100644 --- a/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent +++ b/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent @@ -15,8 +15,6 @@ __addDays __arraySum __isLeapYear _abort -_emscripten_builtin_free -_emscripten_builtin_malloc _emscripten_get_heap_size _emscripten_memcpy_big _emscripten_resize_heap diff --git a/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent b/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent index 8f59a5e7a9d9f..0bd014466e233 100644 --- a/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent +++ b/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent @@ -641,6 +641,7 @@ emscripten_autodebug_i16 emscripten_autodebug_i32 emscripten_autodebug_i64 emscripten_autodebug_i8 +emscripten_builtin_memalign emscripten_call_worker emscripten_cancel_animation_frame emscripten_cancel_main_loop diff --git a/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent b/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent index 679ad204f7316..be5b5ffa0a287 100644 --- a/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent +++ b/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent @@ -638,8 +638,6 @@ _emscripten_autodebug_i16 _emscripten_autodebug_i32 _emscripten_autodebug_i64 _emscripten_autodebug_i8 -_emscripten_builtin_free -_emscripten_builtin_malloc _emscripten_builtin_memalign _emscripten_call_worker _emscripten_cancel_animation_frame diff --git a/tests/test_other.py b/tests/test_other.py index d300a2d25e942..d469475c1ed31 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7956,9 +7956,9 @@ def args_to_filename(args): self.assertEqual(funcs, expected_funcs) @parameterized({ - 'O0': ([], 11, [], ['waka'], 9211, 5, 14, 16), # noqa - 'O1': (['-O1'], 9, [], ['waka'], 7886, 2, 13, 10), # noqa - 'O2': (['-O2'], 9, [], ['waka'], 7871, 2, 13, 10), # noqa + 'O0': ([], 11, [], ['waka'], 9211, 5, 12, 16), # noqa + 'O1': (['-O1'], 9, [], ['waka'], 7886, 2, 11, 10), # noqa + 'O2': (['-O2'], 9, [], ['waka'], 7871, 2, 11, 10), # noqa # in -O3, -Os and -Oz we metadce, and they shrink it down to the minimal output we want 'O3': (['-O3'], 0, [], [], 85, 0, 2, 2), # noqa 'Os': (['-Os'], 0, [], [], 85, 0, 2, 2), # noqa @@ -7984,13 +7984,13 @@ def test_binaryen_metadce_minimal_fastcomp(self, *args): @no_fastcomp() def test_binaryen_metadce_cxx(self): # test on libc++: see effects of emulated function pointers - self.run_metadce_test('hello_libcxx.cpp', ['-O2'], 32, [], ['waka'], 226582, 21, 34, 559) # noqa + self.run_metadce_test('hello_libcxx.cpp', ['-O2'], 32, [], ['waka'], 226582, 21, 32, 559) # noqa @parameterized({ - 'normal': (['-O2'], 33, ['abort'], ['waka'], 186423, 29, 38, 539), # noqa + 'normal': (['-O2'], 31, ['abort'], ['waka'], 186423, 29, 38, 539), # noqa 'enumated_function_pointers': (['-O2', '-s', 'EMULATED_FUNCTION_POINTERS=1'], - 33, ['abort'], ['waka'], 186423, 29, 39, 519), # noqa + 31, ['abort'], ['waka'], 186423, 29, 39, 519), # noqa }) @no_wasm_backend() def test_binaryen_metadce_cxx_fastcomp(self, *args): @@ -7998,9 +7998,9 @@ def test_binaryen_metadce_cxx_fastcomp(self, *args): self.run_metadce_test('hello_libcxx.cpp', *args) @parameterized({ - 'O0': ([], 17, [], ['waka'], 22185, 11, 19, 57), # noqa - 'O1': (['-O1'], 15, [], ['waka'], 10415, 9, 16, 31), # noqa - 'O2': (['-O2'], 15, [], ['waka'], 10183, 9, 16, 25), # noqa + 'O0': ([], 17, [], ['waka'], 22185, 11, 17, 57), # noqa + 'O1': (['-O1'], 15, [], ['waka'], 10415, 9, 14, 31), # noqa + 'O2': (['-O2'], 15, [], ['waka'], 10183, 9, 14, 25), # noqa 'O3': (['-O3'], 5, [], [], 2353, 7, 3, 14), # noqa; in -O3, -Os and -Oz we metadce 'Os': (['-Os'], 5, [], [], 2310, 7, 3, 15), # noqa 'Oz': (['-Oz'], 5, [], [], 2272, 7, 2, 14), # noqa @@ -8012,7 +8012,7 @@ def test_binaryen_metadce_cxx_fastcomp(self, *args): # don't compare the # of functions in a main module, which changes a lot # TODO(sbc): Investivate why the number of exports is order of magnitude # larger for wasm backend. - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1605, [], [], 517336, 172, 1506, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1606, [], [], 517336, 172, 1485, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10770, 17, 13, None), # noqa }) @no_fastcomp() @@ -8032,7 +8032,7 @@ def test_binaryen_metadce_hello(self, *args): 0, [], [], 8, 0, 0, 0), # noqa; totally empty! # we don't metadce with linkable code! other modules may want stuff # don't compare the # of functions in a main module, which changes a lot - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1570, [], [], 226403, 30, 96, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1568, [], [], 226403, 30, 96, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10571, 19, 9, 21), # noqa }) @no_wasm_backend() From 601f36fef7b4b08d30b153b9dfb0acb01a5d6975 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 24 Jun 2019 14:01:27 -0700 Subject: [PATCH 43/62] Fix other.test_binaryen_names --- tests/test_other.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index d469475c1ed31..36143eed1bb34 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7664,20 +7664,12 @@ def test_binaryen_names(self): # we use dlmalloc here, as emmalloc has a bunch of asserts that contain the text "malloc" in them, which makes counting harder run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp')] + args + ['-s', 'MALLOC="dlmalloc"']) code = open('a.out.wasm', 'rb').read() - if self.is_wasm_backend(): - if expect_names: - # name section adds the name of malloc (there is also another one for the exports: malloc and emscripten_builtin_malloc) - self.assertEqual(code.count(b'malloc'), 3) - else: - # should be just malloc and emscripten_builtin_malloc, for the export - self.assertEqual(code.count(b'malloc'), 2) + if expect_names: + # name section adds the name of malloc (there is also another one for the export) + self.assertEqual(code.count(b'malloc'), 2) else: - if expect_names: - # name section adds the name of malloc (there is also another one for the export) - self.assertEqual(code.count(b'malloc'), 2) - else: - # should be just malloc for the export - self.assertEqual(code.count(b'malloc'), 1) + # should be just malloc for the export + self.assertEqual(code.count(b'malloc'), 1) sizes[str(args)] = os.path.getsize('a.out.wasm') print(sizes) self.assertLess(sizes["['-O2']"], sizes["['-O2', '--profiling-funcs']"], 'when -profiling-funcs, the size increases due to function names') From 186065394a16815e3fed231136e5daca499dd041 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 24 Jun 2019 14:15:15 -0700 Subject: [PATCH 44/62] Only use _emscripten_builtin_memalign/free with LSan --- src/library_syscall.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library_syscall.js b/src/library_syscall.js index 68b337812069f..1c422aa906dcc 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -455,7 +455,7 @@ var SyscallsLibrary = { FS.munmap(stream); SYSCALLS.mappings[addr] = null; if (info.allocated) { -#if WASM_BACKEND +#if USE_LSAN _emscripten_builtin_free(info.malloc); #else _free(info.malloc); @@ -986,7 +986,7 @@ var SyscallsLibrary = { var ptr; var allocated = false; if (fd === -1) { -#if WASM_BACKEND +#if USE_LSAN ptr = _emscripten_builtin_memalign(PAGE_SIZE, len); #else ptr = _memalign(PAGE_SIZE, len); From 973666aabc9f42b1bd4aa8fa4a6cf484817642ff Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 24 Jun 2019 15:10:07 -0700 Subject: [PATCH 45/62] Fix metadce tests more --- tests/other/metadce/hello_world_O3_MAIN_MODULE.sent | 1 - tests/test_other.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent b/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent index 0bd014466e233..8f59a5e7a9d9f 100644 --- a/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent +++ b/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent @@ -641,7 +641,6 @@ emscripten_autodebug_i16 emscripten_autodebug_i32 emscripten_autodebug_i64 emscripten_autodebug_i8 -emscripten_builtin_memalign emscripten_call_worker emscripten_cancel_animation_frame emscripten_cancel_main_loop diff --git a/tests/test_other.py b/tests/test_other.py index 36143eed1bb34..08ea6c2e6ae7e 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8004,7 +8004,7 @@ def test_binaryen_metadce_cxx_fastcomp(self, *args): # don't compare the # of functions in a main module, which changes a lot # TODO(sbc): Investivate why the number of exports is order of magnitude # larger for wasm backend. - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1606, [], [], 517336, 172, 1485, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1605, [], [], 517336, 172, 1485, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10770, 17, 13, None), # noqa }) @no_fastcomp() From 3bf391ed3bcf2400e43f1d4e6b43e89dff96a9c2 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 24 Jun 2019 15:52:15 -0700 Subject: [PATCH 46/62] Use self.assertTrue in do_smart_test --- tests/test_other.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index 08ea6c2e6ae7e..54f08e5a113b4 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -149,8 +149,7 @@ def do_smart_test(self, source, name='test.cpp', literals=[], regexes=[], self.assertContained([literal], seen) for regex in regexes: - if not re.search(regex, seen): - self.fail('Expected regex "%s" to match on:\n%s' % (regex, seen)) + self.assertTrue(re.search(regex, seen), 'Expected regex "%s" to match on:\n%s' % (regex, seen)) def run_on_pty(self, cmd): master, slave = os.openpty() From 21da2e7db67361ba73cba06e30e1139f19680dd0 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 24 Jun 2019 16:24:31 -0700 Subject: [PATCH 47/62] metadce tests are evil --- tests/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index 54f08e5a113b4..c157926081fd3 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8003,7 +8003,7 @@ def test_binaryen_metadce_cxx_fastcomp(self, *args): # don't compare the # of functions in a main module, which changes a lot # TODO(sbc): Investivate why the number of exports is order of magnitude # larger for wasm backend. - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1605, [], [], 517336, 172, 1485, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1605, [], [], 517336, 172, 1507, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10770, 17, 13, None), # noqa }) @no_fastcomp() From 7208374e62d077b49a46b6674cf62c573cb47251 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Mon, 24 Jun 2019 17:32:54 -0700 Subject: [PATCH 48/62] Fix metadce tests harder --- tests/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index c157926081fd3..e93a8566e5f88 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8003,7 +8003,7 @@ def test_binaryen_metadce_cxx_fastcomp(self, *args): # don't compare the # of functions in a main module, which changes a lot # TODO(sbc): Investivate why the number of exports is order of magnitude # larger for wasm backend. - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1605, [], [], 517336, 172, 1507, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1605, [], [], 517336, 172, 1506, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10770, 17, 13, None), # noqa }) @no_fastcomp() From 01f5f6f63e2e89523f26207a43f565e11a62c4cc Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Tue, 25 Jun 2019 13:23:31 -0700 Subject: [PATCH 49/62] Fix munmap to not dlfree unless mmap used dlmalloc --- src/library_syscall.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/library_syscall.js b/src/library_syscall.js index 1c422aa906dcc..28e79e88b9999 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -456,7 +456,11 @@ var SyscallsLibrary = { SYSCALLS.mappings[addr] = null; if (info.allocated) { #if USE_LSAN - _emscripten_builtin_free(info.malloc); + if (info.fd === -1) { + _emscripten_builtin_free(info.malloc); + } else { + _free(info.malloc); + } #else _free(info.malloc); #endif From e2d61255cfd52996b7e89bfd2716f94cfd1f2250 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 26 Jun 2019 13:24:37 -0700 Subject: [PATCH 50/62] Add comment for mmap/munmap changes --- src/library_syscall.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/library_syscall.js b/src/library_syscall.js index f6b81a1f77dfc..ac100e1ab9e70 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -459,6 +459,9 @@ var SyscallsLibrary = { SYSCALLS.mappings[addr] = null; if (info.allocated) { #if USE_LSAN + // The memory allocator in sanitizers uses MAP_ANONYMOUS to allocate memory. + // To avoid calling back into the sanitizer memory allocator, we use the builtin memalign. + // Therefore, if we are using sanitizer and MAP_ANONYMOUS, the memory must be freed with the builtin free. if (info.anonymous) { _emscripten_builtin_free(info.malloc); } else { @@ -1022,6 +1025,9 @@ var SyscallsLibrary = { // In this case fd argument is ignored. if ((flags & {{{ cDefine('MAP_ANONYMOUS') }}}) !== 0) { #if USE_LSAN + // The memory allocator in sanitizers uses MAP_ANONYMOUS to allocate memory. + // It is very important that we call the builtin version of memalign in this instance, + // for otherwise the memalign call would call back into the sanitizer's memory allocator. ptr = _emscripten_builtin_memalign(PAGE_SIZE, len); #else ptr = _memalign(PAGE_SIZE, len); From 087c2d0546500ed8bfd26c0f05195db43144e249 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 26 Jun 2019 13:29:06 -0700 Subject: [PATCH 51/62] Remove old workaround in emmalloc.cpp --- system/lib/emmalloc.cpp | 2 +- tests/test_core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/system/lib/emmalloc.cpp b/system/lib/emmalloc.cpp index 58ca9c936c250..3814cdf214603 100644 --- a/system/lib/emmalloc.cpp +++ b/system/lib/emmalloc.cpp @@ -1193,7 +1193,7 @@ struct mallinfo mallinfo() { // in their code, and make those replacements refer to the original malloc // and free from this file. // This allows an easy mechanism for hooking into memory allocation. -#if defined(__EMSCRIPTEN__) && !defined(TESTING_EMMALLOC) +#if defined(__EMSCRIPTEN__) extern __typeof(malloc) emscripten_builtin_malloc __attribute__((alias("malloc"))); extern __typeof(free) emscripten_builtin_free __attribute__((alias("free"))); extern __typeof(memalign) emscripten_builtin_memalign __attribute__((alias("memalign"))); diff --git a/tests/test_core.py b/tests/test_core.py index 8284e4330a85e..f9f133aadcb4c 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -759,7 +759,7 @@ def test_emmalloc(self, *args): # in newer clang+llvm, the internal calls to malloc in emmalloc may be optimized under # the assumption that they are external, so like in system_libs.py where we build # malloc, we need to disable builtin here too - self.emcc_args += ['-fno-builtin', '-DTESTING_EMMALLOC'] + list(args) + self.emcc_args += ['-fno-builtin'] + list(args) self.do_run(open(path_from_root('system', 'lib', 'emmalloc.cpp')).read() + open(path_from_root('tests', 'core', 'test_emmalloc.cpp')).read(), open(path_from_root('tests', 'core', 'test_emmalloc.txt')).read()) From fb3a71399780cbc901f6ac523cdb12449d3f4b4b Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 26 Jun 2019 14:19:02 -0700 Subject: [PATCH 52/62] Move builtin memory allocation logic into one place --- src/library.js | 47 +++++++- src/library_syscall.js | 110 ++++++++---------- src/runtime_strings.js | 4 +- system/lib/compiler-rt/lib/lsan/lsan.cc | 5 +- .../sanitizer_common/sanitizer_emscripten.cc | 16 +++ .../lib/sanitizer_common/sanitizer_linux.cc | 8 +- .../lib/compiler-rt/lib/ubsan/ubsan_flags.cc | 5 +- 7 files changed, 118 insertions(+), 77 deletions(-) diff --git a/src/library.js b/src/library.js index 4751fd8df8bf7..d0c33bd416354 100644 --- a/src/library.js +++ b/src/library.js @@ -4522,7 +4522,7 @@ LibraryManager.library = { }, // Look up the function name from our stack frame cache with our PC representation. - emscripten_pc_get_function__deps: ['$UNWIND_CACHE'], + emscripten_pc_get_function__deps: ['$UNWIND_CACHE', 'emscripten_with_builtin_malloc'], emscripten_pc_get_function: function (pc) { var frame = UNWIND_CACHE[pc]; if (!frame) return 0; @@ -4536,8 +4536,10 @@ LibraryManager.library = { } if (!name) return 0; - if (_emscripten_pc_get_function.ret) _emscripten_builtin_free(_emscripten_pc_get_function.ret); - _emscripten_pc_get_function.ret = allocateUTF8(name, _emscripten_builtin_malloc); + _emscripten_with_builtin_malloc(function () { + if (_emscripten_pc_get_function.ret) _free(_emscripten_pc_get_function.ret); + _emscripten_pc_get_function.ret = allocateUTF8(name); + }); return _emscripten_pc_get_function.ret; }, @@ -4573,13 +4575,15 @@ LibraryManager.library = { }, // Look up the file name from our stack frame cache with our PC representation. - emscripten_pc_get_file__deps: ['emscripten_pc_get_source_js'], + emscripten_pc_get_file__deps: ['emscripten_pc_get_source_js', 'emscripten_with_builtin_malloc'], emscripten_pc_get_file: function (pc) { var result = _emscripten_pc_get_source_js(pc); if (!result) return 0; - if (_emscripten_pc_get_file.ret) _emscripten_builtin_free(_emscripten_pc_get_file.ret); - _emscripten_pc_get_file.ret = allocateUTF8(result.file, _emscripten_builtin_malloc); + _emscripten_with_builtin_malloc(function () { + if (_emscripten_pc_get_file.ret) _free(_emscripten_pc_get_file.ret); + _emscripten_pc_get_file.ret = allocateUTF8(result.file); + }); return _emscripten_pc_get_file.ret; }, @@ -4601,6 +4605,37 @@ LibraryManager.library = { return stringToUTF8(wasmBinaryFile, buf, length); }, + emscripten_with_builtin_malloc__deps: ['emscripten_builtin_malloc'], + emscripten_with_builtin_malloc: function (func) { + var prev_malloc = _malloc; + var prev_memalign = _memalign; + var prev_free = _free; + _malloc = _emscripten_builtin_malloc; + _memalign = _emscripten_builtin_memalign; + _free = _emscripten_builtin_free; + try { + return func(); + } finally { + _malloc = prev_malloc; + _memalign = prev_memalign; + _free = prev_free; + } + }, + + emscripten_builtin_mmap2__deps: ['emscripten_with_builtin_malloc', '$SYSCALLS'], + emscripten_builtin_mmap2: function (addr, len, prot, flags, fd, off) { + return _emscripten_with_builtin_malloc(function () { + return SYSCALLS.doMmap2(addr, len, prot, flags, fd, off); + }); + }, + + emscripten_builtin_munmap__deps: ['emscripten_with_builtin_malloc', '$SYSCALLS'], + emscripten_builtin_munmap: function (addr, len) { + return _emscripten_with_builtin_malloc(function () { + return SYSCALLS.doMunmap(addr, len); + }); + }, + emscripten_get_stack_top: function() { return STACKTOP; }, diff --git a/src/library_syscall.js b/src/library_syscall.js index ac100e1ab9e70..9f658fc4f0e6a 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -13,6 +13,52 @@ var SyscallsLibrary = { #endif ], $SYSCALLS: { + doMmap2: function(addr, len, prot, flags, fd, off) { + off <<= 12; // undo pgoffset + var ptr; + var allocated = false; + + // addr argument must be page aligned if MAP_FIXED flag is set. + if ((flags & {{{ cDefine('MAP_FIXED') }}}) !== 0 && (addr % PAGE_SIZE) !== 0) { + return -{{{ cDefine('EINVAL') }}}; + } + + // MAP_ANONYMOUS (aka MAP_ANON) isn't actually defined by POSIX spec, + // but it is widely used way to allocate memory pages on Linux, BSD and Mac. + // In this case fd argument is ignored. + if ((flags & {{{ cDefine('MAP_ANONYMOUS') }}}) !== 0) { + ptr = _memalign(PAGE_SIZE, len); + if (!ptr) return -{{{ cDefine('ENOMEM') }}}; + _memset(ptr, 0, len); + allocated = true; + } else { + var info = FS.getStream(fd); + if (!info) return -{{{ cDefine('EBADF') }}}; + var res = FS.mmap(info, HEAPU8, addr, len, off, prot, flags); + ptr = res.ptr; + allocated = res.allocated; + } + SYSCALLS.mappings[ptr] = { malloc: ptr, len: len, allocated: allocated, fd: fd, flags: flags }; + return ptr; + }, + doMunmap: function(addr, len) { + if (addr == {{{ cDefine('MAP_FAILED') }}} || len == 0) { + return -{{{ cDefine('EINVAL') }}}; + } + // TODO: support unmmap'ing parts of allocations + var info = SYSCALLS.mappings[addr]; + if (!info) return 0; + if (len === info.len) { + var stream = FS.getStream(info.fd); + SYSCALLS.doMsync(addr, stream, len, info.flags) + FS.munmap(stream); + SYSCALLS.mappings[addr] = null; + if (info.allocated) { + _free(info.malloc); + } + } + return 0; + }, #if SYSCALLS_REQUIRE_FILESYSTEM // global constants DEFAULT_POLLMASK: {{{ cDefine('POLLIN') }}} | {{{ cDefine('POLLOUT') }}}, @@ -446,33 +492,7 @@ var SyscallsLibrary = { }, __syscall91: function(which, varargs) { // munmap var addr = SYSCALLS.get(), len = SYSCALLS.get(); - if (addr == {{{ cDefine('MAP_FAILED') }}} || len == 0) { - return -{{{ cDefine('EINVAL') }}}; - } - // TODO: support unmmap'ing parts of allocations - var info = SYSCALLS.mappings[addr]; - if (!info) return 0; - if (len === info.len) { - var stream = FS.getStream(info.fd); - SYSCALLS.doMsync(addr, stream, len, info.flags) - FS.munmap(stream); - SYSCALLS.mappings[addr] = null; - if (info.allocated) { -#if USE_LSAN - // The memory allocator in sanitizers uses MAP_ANONYMOUS to allocate memory. - // To avoid calling back into the sanitizer memory allocator, we use the builtin memalign. - // Therefore, if we are using sanitizer and MAP_ANONYMOUS, the memory must be freed with the builtin free. - if (info.anonymous) { - _emscripten_builtin_free(info.malloc); - } else { - _free(info.malloc); - } -#else - _free(info.malloc); -#endif - } - } - return 0; + return SYSCALLS.doMunmap(addr, len); }, __syscall94: function(which, varargs) { // fchmod var fd = SYSCALLS.get(), mode = SYSCALLS.get(); @@ -1010,41 +1030,7 @@ var SyscallsLibrary = { __syscall192__deps: ['emscripten_builtin_memalign', 'memalign'], __syscall192: function(which, varargs) { // mmap2 var addr = SYSCALLS.get(), len = SYSCALLS.get(), prot = SYSCALLS.get(), flags = SYSCALLS.get(), fd = SYSCALLS.get(), off = SYSCALLS.get() - off <<= 12; // undo pgoffset - var ptr; - var allocated = false; - var anonymous = false; - - // addr argument must be page aligned if MAP_FIXED flag is set. - if ((flags & {{{ cDefine('MAP_FIXED') }}}) !== 0 && (addr % PAGE_SIZE) !== 0) { - return -{{{ cDefine('EINVAL') }}}; - } - - // MAP_ANONYMOUS (aka MAP_ANON) isn't actually defined by POSIX spec, - // but it is widely used way to allocate memory pages on Linux, BSD and Mac. - // In this case fd argument is ignored. - if ((flags & {{{ cDefine('MAP_ANONYMOUS') }}}) !== 0) { -#if USE_LSAN - // The memory allocator in sanitizers uses MAP_ANONYMOUS to allocate memory. - // It is very important that we call the builtin version of memalign in this instance, - // for otherwise the memalign call would call back into the sanitizer's memory allocator. - ptr = _emscripten_builtin_memalign(PAGE_SIZE, len); -#else - ptr = _memalign(PAGE_SIZE, len); -#endif - if (!ptr) return -{{{ cDefine('ENOMEM') }}}; - _memset(ptr, 0, len); - allocated = true; - anonymous = true; - } else { - var info = FS.getStream(fd); - if (!info) return -{{{ cDefine('EBADF') }}}; - var res = FS.mmap(info, HEAPU8, addr, len, off, prot, flags); - ptr = res.ptr; - allocated = res.allocated; - } - SYSCALLS.mappings[ptr] = { malloc: ptr, len: len, allocated: allocated, fd: fd, flags: flags, anonymous: anonymous }; - return ptr; + return SYSCALLS.doMmap2(addr, len, prot, flags, fd, off); }, __syscall193: function(which, varargs) { // truncate64 var path = SYSCALLS.getStr(), zero = SYSCALLS.getZero(), length = SYSCALLS.get64(); diff --git a/src/runtime_strings.js b/src/runtime_strings.js index bbb7f147f929f..6c0b61374e21f 100644 --- a/src/runtime_strings.js +++ b/src/runtime_strings.js @@ -376,9 +376,9 @@ function lengthBytesUTF32(str) { // Allocate heap space for a JS string, and write it there. // It is the responsibility of the caller to free() that memory. -function allocateUTF8(str, allocator) { +function allocateUTF8(str) { var size = lengthBytesUTF8(str) + 1; - var ret = (allocator || _malloc)(size); + var ret = _malloc(size); if (ret) stringToUTF8Array(str, HEAP8, ret, size); return ret; } diff --git a/system/lib/compiler-rt/lib/lsan/lsan.cc b/system/lib/compiler-rt/lib/lsan/lsan.cc index 021e3c9f36035..571512f4224aa 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan.cc @@ -70,8 +70,9 @@ static void InitializeFlags() { parser.ParseString(lsan_default_options); #if SANITIZER_EMSCRIPTEN char *options = (char*) EM_ASM_INT({ - return allocateUTF8(Module['LSAN_OPTIONS'] || 0, - _emscripten_builtin_malloc); + return _emscripten_with_builtin_malloc(function () { + return allocateUTF8(Module['LSAN_OPTIONS'] || 0); + }); }); parser.ParseString(options); emscripten_builtin_free(options); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc index f9774724f76cd..bb8ebf9527bba 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc @@ -67,6 +67,22 @@ int internal_sigaction(int signum, const void *act, void *oldact) { (struct sigaction *)oldact); } +extern "C" { + uptr emscripten_builtin_mmap2(void *addr, uptr length, int prot, int flags, + int fd, unsigned offset); + uptr emscripten_builtin_munmap(void *addr, uptr length); +} + +uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, + OFF_T offset) { + CHECK(IsAligned(offset, 4096)); + return emscripten_builtin_mmap2(addr, length, prot, flags, fd, offset / 4096); +} + +uptr internal_munmap(void *addr, uptr length) { + return emscripten_builtin_munmap(addr, length); +} + extern "C" uptr emscripten_get_stack_top(); extern "C" uptr emscripten_get_stack_base(); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc index c14fc2ac3ad79..701ba01982291 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc @@ -174,7 +174,7 @@ namespace __sanitizer { // --------------- sanitizer_libc.h #if !SANITIZER_SOLARIS && !SANITIZER_NETBSD -#if !SANITIZER_S390 && !SANITIZER_OPENBSD +#if !SANITIZER_S390 && !SANITIZER_OPENBSD && !SANITIZER_EMSCRIPTEN uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, OFF_T offset) { #if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS @@ -187,13 +187,15 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, offset / 4096); #endif } -#endif // !SANITIZER_S390 && !SANITIZER_OPENBSD +#endif // !SANITIZER_S390 && !SANITIZER_OPENBSD && !SANITIZER_NETBSD -#if !SANITIZER_OPENBSD +#if !SANITIZER_OPENBSD && !SANITIZER_EMSCRIPTEN uptr internal_munmap(void *addr, uptr length) { return internal_syscall(SYSCALL(munmap), (uptr)addr, length); } +#endif +#if !SANITIZER_OPENBSD int internal_mprotect(void *addr, uptr length, int prot) { return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot); } diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_flags.cc b/system/lib/compiler-rt/lib/ubsan/ubsan_flags.cc index ab9b966c001f0..f2032081caadb 100644 --- a/system/lib/compiler-rt/lib/ubsan/ubsan_flags.cc +++ b/system/lib/compiler-rt/lib/ubsan/ubsan_flags.cc @@ -82,8 +82,9 @@ void InitializeFlags() { // Override from environment variable. #if SANITIZER_EMSCRIPTEN char *options = (char*) EM_ASM_INT({ - return allocateUTF8(Module['UBSAN_OPTIONS'] || 0, - _emscripten_builtin_malloc); + return _emscripten_with_builtin_malloc(function () { + return allocateUTF8(Module['UBSAN_OPTIONS'] || 0); + }); }); parser.ParseString(options); emscripten_builtin_free(options); From f738e8c19bf9955ac7abd5a421383d5ccddb0829 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 26 Jun 2019 14:24:09 -0700 Subject: [PATCH 53/62] Fix metadce tests --- src/library.js | 2 +- tests/other/metadce/hello_world_O3_MAIN_MODULE.sent | 3 +++ tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent | 5 +++++ tests/test_other.py | 4 ++-- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/library.js b/src/library.js index d0c33bd416354..702cdb526ad09 100644 --- a/src/library.js +++ b/src/library.js @@ -4605,7 +4605,7 @@ LibraryManager.library = { return stringToUTF8(wasmBinaryFile, buf, length); }, - emscripten_with_builtin_malloc__deps: ['emscripten_builtin_malloc'], + emscripten_with_builtin_malloc__deps: ['emscripten_builtin_malloc', 'emscripten_builtin_free', 'emscripten_builtin_memalign'], emscripten_with_builtin_malloc: function (func) { var prev_malloc = _malloc; var prev_memalign = _memalign; diff --git a/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent b/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent index 8f59a5e7a9d9f..759d30fb90bc3 100644 --- a/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent +++ b/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent @@ -641,6 +641,8 @@ emscripten_autodebug_i16 emscripten_autodebug_i32 emscripten_autodebug_i64 emscripten_autodebug_i8 +emscripten_builtin_mmap2 +emscripten_builtin_munmap emscripten_call_worker emscripten_cancel_animation_frame emscripten_cancel_main_loop @@ -1024,6 +1026,7 @@ emscripten_webgl_init_context_attributes emscripten_webgl_make_context_current emscripten_wget emscripten_wget_data +emscripten_with_builtin_malloc emscripten_worker_respond emscripten_worker_respond_provisionally emscripten_yield diff --git a/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent b/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent index be5b5ffa0a287..fc5cba8dda2be 100644 --- a/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent +++ b/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent @@ -638,7 +638,11 @@ _emscripten_autodebug_i16 _emscripten_autodebug_i32 _emscripten_autodebug_i64 _emscripten_autodebug_i8 +_emscripten_builtin_free +_emscripten_builtin_malloc _emscripten_builtin_memalign +_emscripten_builtin_mmap2 +_emscripten_builtin_munmap _emscripten_call_worker _emscripten_cancel_animation_frame _emscripten_cancel_main_loop @@ -1021,6 +1025,7 @@ _emscripten_webgl_init_context_attributes _emscripten_webgl_make_context_current _emscripten_wget _emscripten_wget_data +_emscripten_with_builtin_malloc _emscripten_worker_respond _emscripten_worker_respond_provisionally _emscripten_yield diff --git a/tests/test_other.py b/tests/test_other.py index 3d18c1cb5d932..474401a46691e 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8023,7 +8023,7 @@ def test_binaryen_metadce_cxx_fastcomp(self, *args): # don't compare the # of functions in a main module, which changes a lot # TODO(sbc): Investivate why the number of exports is order of magnitude # larger for wasm backend. - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1605, [], [], 517336, 172, 1506, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1608, [], [], 517336, 172, 1507, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10770, 17, 13, None), # noqa }) @no_fastcomp() @@ -8043,7 +8043,7 @@ def test_binaryen_metadce_hello(self, *args): 0, [], [], 8, 0, 0, 0), # noqa; totally empty! # we don't metadce with linkable code! other modules may want stuff # don't compare the # of functions in a main module, which changes a lot - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1568, [], [], 226403, 30, 96, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1573, [], [], 226403, 30, 96, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10571, 19, 9, 21), # noqa }) @no_wasm_backend() From 0b958c02a295c74d315877c4846bcc0f64ec8b19 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 26 Jun 2019 15:25:08 -0700 Subject: [PATCH 54/62] Fix mmap/munmap dependecy issue --- src/library.js | 8 ++-- src/library_syscall.js | 103 +++++++++++++++++++++-------------------- 2 files changed, 58 insertions(+), 53 deletions(-) diff --git a/src/library.js b/src/library.js index 702cdb526ad09..c171aee8351c8 100644 --- a/src/library.js +++ b/src/library.js @@ -4622,17 +4622,17 @@ LibraryManager.library = { } }, - emscripten_builtin_mmap2__deps: ['emscripten_with_builtin_malloc', '$SYSCALLS'], + emscripten_builtin_mmap2__deps: ['emscripten_with_builtin_malloc', '_emscripten_syscall_mmap2'], emscripten_builtin_mmap2: function (addr, len, prot, flags, fd, off) { return _emscripten_with_builtin_malloc(function () { - return SYSCALLS.doMmap2(addr, len, prot, flags, fd, off); + return __emscripten_syscall_mmap2(addr, len, prot, flags, fd, off); }); }, - emscripten_builtin_munmap__deps: ['emscripten_with_builtin_malloc', '$SYSCALLS'], + emscripten_builtin_munmap__deps: ['emscripten_with_builtin_malloc', '_emscripten_syscall_munmap'], emscripten_builtin_munmap: function (addr, len) { return _emscripten_with_builtin_malloc(function () { - return SYSCALLS.doMunmap(addr, len); + return __emscripten_syscall_munmap(addr, len); }); }, diff --git a/src/library_syscall.js b/src/library_syscall.js index 9f658fc4f0e6a..12d86d3aaab9b 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -13,52 +13,6 @@ var SyscallsLibrary = { #endif ], $SYSCALLS: { - doMmap2: function(addr, len, prot, flags, fd, off) { - off <<= 12; // undo pgoffset - var ptr; - var allocated = false; - - // addr argument must be page aligned if MAP_FIXED flag is set. - if ((flags & {{{ cDefine('MAP_FIXED') }}}) !== 0 && (addr % PAGE_SIZE) !== 0) { - return -{{{ cDefine('EINVAL') }}}; - } - - // MAP_ANONYMOUS (aka MAP_ANON) isn't actually defined by POSIX spec, - // but it is widely used way to allocate memory pages on Linux, BSD and Mac. - // In this case fd argument is ignored. - if ((flags & {{{ cDefine('MAP_ANONYMOUS') }}}) !== 0) { - ptr = _memalign(PAGE_SIZE, len); - if (!ptr) return -{{{ cDefine('ENOMEM') }}}; - _memset(ptr, 0, len); - allocated = true; - } else { - var info = FS.getStream(fd); - if (!info) return -{{{ cDefine('EBADF') }}}; - var res = FS.mmap(info, HEAPU8, addr, len, off, prot, flags); - ptr = res.ptr; - allocated = res.allocated; - } - SYSCALLS.mappings[ptr] = { malloc: ptr, len: len, allocated: allocated, fd: fd, flags: flags }; - return ptr; - }, - doMunmap: function(addr, len) { - if (addr == {{{ cDefine('MAP_FAILED') }}} || len == 0) { - return -{{{ cDefine('EINVAL') }}}; - } - // TODO: support unmmap'ing parts of allocations - var info = SYSCALLS.mappings[addr]; - if (!info) return 0; - if (len === info.len) { - var stream = FS.getStream(info.fd); - SYSCALLS.doMsync(addr, stream, len, info.flags) - FS.munmap(stream); - SYSCALLS.mappings[addr] = null; - if (info.allocated) { - _free(info.malloc); - } - } - return 0; - }, #if SYSCALLS_REQUIRE_FILESYSTEM // global constants DEFAULT_POLLMASK: {{{ cDefine('POLLIN') }}} | {{{ cDefine('POLLOUT') }}}, @@ -265,6 +219,56 @@ var SyscallsLibrary = { } }, + _emscripten_syscall_mmap2__deps: ['memalign', 'memset', '$SYSCALLS', '$FS'], + _emscripten_syscall_mmap2: function(addr, len, prot, flags, fd, off) { + off <<= 12; // undo pgoffset + var ptr; + var allocated = false; + + // addr argument must be page aligned if MAP_FIXED flag is set. + if ((flags & {{{ cDefine('MAP_FIXED') }}}) !== 0 && (addr % PAGE_SIZE) !== 0) { + return -{{{ cDefine('EINVAL') }}}; + } + + // MAP_ANONYMOUS (aka MAP_ANON) isn't actually defined by POSIX spec, + // but it is widely used way to allocate memory pages on Linux, BSD and Mac. + // In this case fd argument is ignored. + if ((flags & {{{ cDefine('MAP_ANONYMOUS') }}}) !== 0) { + ptr = _memalign(PAGE_SIZE, len); + if (!ptr) return -{{{ cDefine('ENOMEM') }}}; + _memset(ptr, 0, len); + allocated = true; + } else { + var info = FS.getStream(fd); + if (!info) return -{{{ cDefine('EBADF') }}}; + var res = FS.mmap(info, HEAPU8, addr, len, off, prot, flags); + ptr = res.ptr; + allocated = res.allocated; + } + SYSCALLS.mappings[ptr] = { malloc: ptr, len: len, allocated: allocated, fd: fd, flags: flags }; + return ptr; + }, + + _emscripten_syscall_munmap__deps: ['$SYSCALLS', '$FS'], + _emscripten_syscall_munmap: function(addr, len) { + if (addr == {{{ cDefine('MAP_FAILED') }}} || len == 0) { + return -{{{ cDefine('EINVAL') }}}; + } + // TODO: support unmmap'ing parts of allocations + var info = SYSCALLS.mappings[addr]; + if (!info) return 0; + if (len === info.len) { + var stream = FS.getStream(info.fd); + SYSCALLS.doMsync(addr, stream, len, info.flags) + FS.munmap(stream); + SYSCALLS.mappings[addr] = null; + if (info.allocated) { + _free(info.malloc); + } + } + return 0; + }, + __syscall1: function(which, varargs) { // exit var status = SYSCALLS.get(); exit(status); @@ -490,9 +494,10 @@ var SyscallsLibrary = { var path = SYSCALLS.getStr(), buf = SYSCALLS.get(), bufsize = SYSCALLS.get(); return SYSCALLS.doReadlink(path, buf, bufsize); }, + __syscall91__deps: ['_emscripten_syscall_munmap'], __syscall91: function(which, varargs) { // munmap var addr = SYSCALLS.get(), len = SYSCALLS.get(); - return SYSCALLS.doMunmap(addr, len); + return __emscripten_syscall_munmap(addr, len); }, __syscall94: function(which, varargs) { // fchmod var fd = SYSCALLS.get(), mode = SYSCALLS.get(); @@ -1027,10 +1032,10 @@ var SyscallsLibrary = { {{{ makeSetValue('rlim', C_STRUCTS.rlimit.rlim_max + 4, '-1', 'i32') }}}; // RLIM_INFINITY return 0; // just report no limits }, - __syscall192__deps: ['emscripten_builtin_memalign', 'memalign'], + __syscall192__deps: ['_emscripten_syscall_mmap2'], __syscall192: function(which, varargs) { // mmap2 var addr = SYSCALLS.get(), len = SYSCALLS.get(), prot = SYSCALLS.get(), flags = SYSCALLS.get(), fd = SYSCALLS.get(), off = SYSCALLS.get() - return SYSCALLS.doMmap2(addr, len, prot, flags, fd, off); + return __emscripten_syscall_mmap2(addr, len, prot, flags, fd, off); }, __syscall193: function(which, varargs) { // truncate64 var path = SYSCALLS.getStr(), zero = SYSCALLS.getZero(), length = SYSCALLS.get64(); From 0c08b66546740bec4fde5fa1b0aff98d43fa9108 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 26 Jun 2019 15:25:30 -0700 Subject: [PATCH 55/62] Don't use a builtin malloc in test_emmalloc_* --- system/lib/emmalloc.cpp | 2 +- tests/test_core.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/system/lib/emmalloc.cpp b/system/lib/emmalloc.cpp index 3814cdf214603..58ca9c936c250 100644 --- a/system/lib/emmalloc.cpp +++ b/system/lib/emmalloc.cpp @@ -1193,7 +1193,7 @@ struct mallinfo mallinfo() { // in their code, and make those replacements refer to the original malloc // and free from this file. // This allows an easy mechanism for hooking into memory allocation. -#if defined(__EMSCRIPTEN__) +#if defined(__EMSCRIPTEN__) && !defined(TESTING_EMMALLOC) extern __typeof(malloc) emscripten_builtin_malloc __attribute__((alias("malloc"))); extern __typeof(free) emscripten_builtin_free __attribute__((alias("free"))); extern __typeof(memalign) emscripten_builtin_memalign __attribute__((alias("memalign"))); diff --git a/tests/test_core.py b/tests/test_core.py index f9f133aadcb4c..9ba82424f6b9a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -759,6 +759,7 @@ def test_emmalloc(self, *args): # in newer clang+llvm, the internal calls to malloc in emmalloc may be optimized under # the assumption that they are external, so like in system_libs.py where we build # malloc, we need to disable builtin here too + self.set_setting('MALLOC', 'none') self.emcc_args += ['-fno-builtin'] + list(args) self.do_run(open(path_from_root('system', 'lib', 'emmalloc.cpp')).read() + open(path_from_root('tests', 'core', 'test_emmalloc.cpp')).read(), From 5939bd3ab311491423359cbea3c4d5a6e6b1b07b Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 26 Jun 2019 15:48:44 -0700 Subject: [PATCH 56/62] Fix metadce tests --- tests/other/metadce/hello_world_O3_MAIN_MODULE.sent | 2 ++ tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent | 2 ++ tests/test_other.py | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent b/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent index 759d30fb90bc3..dff652fcd31e3 100644 --- a/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent +++ b/tests/other/metadce/hello_world_O3_MAIN_MODULE.sent @@ -406,6 +406,8 @@ _emscripten_do_request_fullscreen _emscripten_push_main_loop_blocker _emscripten_push_uncounted_main_loop_blocker _emscripten_save_in_unwind_cache +_emscripten_syscall_mmap2 +_emscripten_syscall_munmap _emscripten_traverse_stack _exit _fillBatteryEventData diff --git a/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent b/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent index fc5cba8dda2be..0acdcec3a7add 100644 --- a/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent +++ b/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent @@ -412,6 +412,8 @@ __emscripten_do_request_fullscreen __emscripten_push_main_loop_blocker __emscripten_push_uncounted_main_loop_blocker __emscripten_save_in_unwind_cache +__emscripten_syscall_mmap2 +__emscripten_syscall_munmap __emscripten_traverse_stack __exit __fillBatteryEventData diff --git a/tests/test_other.py b/tests/test_other.py index 474401a46691e..00ed49148c75e 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8023,7 +8023,7 @@ def test_binaryen_metadce_cxx_fastcomp(self, *args): # don't compare the # of functions in a main module, which changes a lot # TODO(sbc): Investivate why the number of exports is order of magnitude # larger for wasm backend. - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1608, [], [], 517336, 172, 1507, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1610, [], [], 517336, 172, 1506, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10770, 17, 13, None), # noqa }) @no_fastcomp() @@ -8043,7 +8043,7 @@ def test_binaryen_metadce_hello(self, *args): 0, [], [], 8, 0, 0, 0), # noqa; totally empty! # we don't metadce with linkable code! other modules may want stuff # don't compare the # of functions in a main module, which changes a lot - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1573, [], [], 226403, 30, 96, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1575, [], [], 226403, 30, 96, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 15, [], [], 10571, 19, 9, 21), # noqa }) @no_wasm_backend() From 35493fd75cc8d168f7d81031e82dd2d1020f7eb7 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 26 Jun 2019 16:11:10 -0700 Subject: [PATCH 57/62] Fix more metadce tests --- tests/other/metadce/hello_libcxx_O2.sent | 1 + tests/test_other.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/other/metadce/hello_libcxx_O2.sent b/tests/other/metadce/hello_libcxx_O2.sent index 0b34f7dace347..3cb5f6b8f4ecc 100644 --- a/tests/other/metadce/hello_libcxx_O2.sent +++ b/tests/other/metadce/hello_libcxx_O2.sent @@ -13,6 +13,7 @@ __syscall91 __unlock _addDays _arraySum +_emscripten_syscall_munmap _isLeapYear abort abortOnCannotGrowMemory diff --git a/tests/test_other.py b/tests/test_other.py index 00ed49148c75e..40c8e9949be56 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7995,7 +7995,7 @@ def test_binaryen_metadce_minimal_fastcomp(self, *args): @no_fastcomp() def test_binaryen_metadce_cxx(self): # test on libc++: see effects of emulated function pointers - self.run_metadce_test('hello_libcxx.cpp', ['-O2'], 32, [], ['waka'], 226582, 21, 32, 559) # noqa + self.run_metadce_test('hello_libcxx.cpp', ['-O2'], 33, [], ['waka'], 226582, 21, 32, 559) # noqa @parameterized({ 'normal': (['-O2'], 31, ['abort'], ['waka'], 186423, 29, 38, 539), # noqa From 84f264eff823c99609e933f1bc1751b2d824a792 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 26 Jun 2019 16:13:53 -0700 Subject: [PATCH 58/62] Fix no FS test --- src/library_syscall.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/library_syscall.js b/src/library_syscall.js index 12d86d3aaab9b..216348e0751b1 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -219,7 +219,11 @@ var SyscallsLibrary = { } }, - _emscripten_syscall_mmap2__deps: ['memalign', 'memset', '$SYSCALLS', '$FS'], + _emscripten_syscall_mmap2__deps: ['memalign', 'memset', '$SYSCALLS', +#if FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM + '$FS', +#endif + ], _emscripten_syscall_mmap2: function(addr, len, prot, flags, fd, off) { off <<= 12; // undo pgoffset var ptr; @@ -249,7 +253,11 @@ var SyscallsLibrary = { return ptr; }, - _emscripten_syscall_munmap__deps: ['$SYSCALLS', '$FS'], + _emscripten_syscall_munmap__deps: ['$SYSCALLS', +#if FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM + '$FS', +#endif + ], _emscripten_syscall_munmap: function(addr, len) { if (addr == {{{ cDefine('MAP_FAILED') }}} || len == 0) { return -{{{ cDefine('EINVAL') }}}; From eaf0d7477a8eafe7c3e1d427058ac2cc2a44a137 Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Wed, 26 Jun 2019 16:39:25 -0700 Subject: [PATCH 59/62] Fix even more metadce --- tests/other/metadce/hello_libcxx_fastcomp_O2.sent | 1 + .../hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent | 1 + tests/test_other.py | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/other/metadce/hello_libcxx_fastcomp_O2.sent b/tests/other/metadce/hello_libcxx_fastcomp_O2.sent index 2de6935c7b52e..8c58557c720a5 100644 --- a/tests/other/metadce/hello_libcxx_fastcomp_O2.sent +++ b/tests/other/metadce/hello_libcxx_fastcomp_O2.sent @@ -13,6 +13,7 @@ ___syscall91 ___unlock __addDays __arraySum +__emscripten_syscall_munmap __isLeapYear _abort _emscripten_get_heap_size diff --git a/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent b/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent index 2de6935c7b52e..8c58557c720a5 100644 --- a/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent +++ b/tests/other/metadce/hello_libcxx_fastcomp_O2_EMULATED_FUNCTION_POINTERS.sent @@ -13,6 +13,7 @@ ___syscall91 ___unlock __addDays __arraySum +__emscripten_syscall_munmap __isLeapYear _abort _emscripten_get_heap_size diff --git a/tests/test_other.py b/tests/test_other.py index 40c8e9949be56..b31561af41215 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -7998,10 +7998,10 @@ def test_binaryen_metadce_cxx(self): self.run_metadce_test('hello_libcxx.cpp', ['-O2'], 33, [], ['waka'], 226582, 21, 32, 559) # noqa @parameterized({ - 'normal': (['-O2'], 31, ['abort'], ['waka'], 186423, 29, 38, 539), # noqa + 'normal': (['-O2'], 32, ['abort'], ['waka'], 186423, 29, 38, 539), # noqa 'enumated_function_pointers': (['-O2', '-s', 'EMULATED_FUNCTION_POINTERS=1'], - 31, ['abort'], ['waka'], 186423, 29, 39, 519), # noqa + 32, ['abort'], ['waka'], 186423, 29, 39, 519), # noqa }) @no_wasm_backend() def test_binaryen_metadce_cxx_fastcomp(self, *args): From f4e1a5e121c63884815f9e12fa8d9ef047f4c22b Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Thu, 27 Jun 2019 11:38:55 -0700 Subject: [PATCH 60/62] Simplify do_smart_test --- tests/test_other.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/test_other.py b/tests/test_other.py index b31561af41215..891fcd9cd5282 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -138,10 +138,13 @@ def do_other_test(self, dirname, emcc_args=[], run_args=[]): seen = run_js('a.out.js', args=run_args, stderr=PIPE, full_output=True) + '\n' self.assertContained(expected, seen) - def do_smart_test(self, source, name='test.cpp', literals=[], regexes=[], + # Another utility to run a test in this suite. This receives a source file + # to compile, with optional compiler and execution flags. + # Output can be checked by seeing if literals are contained, and that a list + # of regexes match. The return code can also be checked. + def do_smart_test(self, source, literals=[], regexes=[], emcc_args=[], run_args=[], assert_returncode=0): - shutil.copyfile(source, name) - run_process([PYTHON, EMCC, name] + emcc_args) + run_process([PYTHON, EMCC, source] + emcc_args) seen = run_js('a.out.js', args=run_args, stderr=PIPE, full_output=True, assert_returncode=assert_returncode) + '\n' @@ -9335,7 +9338,7 @@ def test_malloc_none(self): def test_lsan_leaks(self, ext): self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_leaks.' + ext), emcc_args=['-fsanitize=leak', '-s', 'ALLOW_MEMORY_GROWTH=1'], - name='test.' + ext, assert_returncode=None, literals=[ + assert_returncode=None, literals=[ 'Direct leak of 2048 byte(s) in 1 object(s) allocated from', 'Direct leak of 1337 byte(s) in 1 object(s) allocated from', 'Direct leak of 42 byte(s) in 1 object(s) allocated from', @@ -9344,24 +9347,24 @@ def test_lsan_leaks(self, ext): @parameterized({ 'c': ['c', [ r'in malloc.*a\.out\.wasm\+0x', - r'(?m)in f /.*/test\.c:6:21$', - r'(?m)in main /.*/test\.c:10:16$', - r'(?m)in main /.*/test\.c:12:3$', - r'(?m)in main /.*/test\.c:13:3$', + r'(?m)in f /.*/test_lsan_leaks\.c:6:21$', + r'(?m)in main /.*/test_lsan_leaks\.c:10:16$', + r'(?m)in main /.*/test_lsan_leaks\.c:12:3$', + r'(?m)in main /.*/test_lsan_leaks\.c:13:3$', ]], 'cpp': ['cpp', [ r'in operator new\[\]\(unsigned long\).*a\.out\.wasm\+0x', - r'(?m)in f\(\) /.*/test\.cpp:4:21$', - r'(?m)in main /.*/test\.cpp:8:16$', - r'(?m)in main /.*/test\.cpp:10:3$', - r'(?m)in main /.*/test\.cpp:11:3$', + r'(?m)in f\(\) /.*/test_lsan_leaks\.cpp:4:21$', + r'(?m)in main /.*/test_lsan_leaks\.cpp:8:16$', + r'(?m)in main /.*/test_lsan_leaks\.cpp:10:3$', + r'(?m)in main /.*/test_lsan_leaks\.cpp:11:3$', ]], }) @no_fastcomp('lsan not supported on fastcomp') def test_lsan_stack_trace(self, ext, regexes): self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_leaks.' + ext), emcc_args=['-fsanitize=leak', '-s', 'ALLOW_MEMORY_GROWTH=1', '-g4'], - name='test.' + ext, assert_returncode=None, literals=[ + assert_returncode=None, literals=[ 'Direct leak of 2048 byte(s) in 1 object(s) allocated from', 'Direct leak of 1337 byte(s) in 1 object(s) allocated from', 'Direct leak of 42 byte(s) in 1 object(s) allocated from', @@ -9375,7 +9378,7 @@ def test_lsan_stack_trace(self, ext, regexes): def test_lsan_no_leak(self, ext): self.do_smart_test(path_from_root('tests', 'other', 'test_lsan_no_leak.' + ext), emcc_args=['-fsanitize=leak', '-s', 'ALLOW_MEMORY_GROWTH=1'], - name='test.' + ext, regexes=[r'^\s*$']) + regexes=[r'^\s*$']) @no_windows('ptys and select are not available on windows') @no_fastcomp('fastcomp clang detects colors differently') From d4dc4c1a6ae35894900a67ce4411fbb15d2e3eab Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Thu, 27 Jun 2019 15:44:19 -0700 Subject: [PATCH 61/62] Remove TESTING_EMMALLOC --- system/lib/emmalloc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/lib/emmalloc.cpp b/system/lib/emmalloc.cpp index 58ca9c936c250..3814cdf214603 100644 --- a/system/lib/emmalloc.cpp +++ b/system/lib/emmalloc.cpp @@ -1193,7 +1193,7 @@ struct mallinfo mallinfo() { // in their code, and make those replacements refer to the original malloc // and free from this file. // This allows an easy mechanism for hooking into memory allocation. -#if defined(__EMSCRIPTEN__) && !defined(TESTING_EMMALLOC) +#if defined(__EMSCRIPTEN__) extern __typeof(malloc) emscripten_builtin_malloc __attribute__((alias("malloc"))); extern __typeof(free) emscripten_builtin_free __attribute__((alias("free"))); extern __typeof(memalign) emscripten_builtin_memalign __attribute__((alias("memalign"))); From a0c81e5c5cfc23c2f93353fcba94e62a154bcf3f Mon Sep 17 00:00:00 2001 From: Guanzhong Chen Date: Thu, 27 Jun 2019 15:46:43 -0700 Subject: [PATCH 62/62] Add some comments about StopTheWorld --- system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc | 2 ++ .../compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc b/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc index 4137256881bf3..512668cd59cdb 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc +++ b/system/lib/compiler-rt/lib/lsan/lsan_emscripten.cc @@ -98,6 +98,8 @@ void HandleLeaks() { } void DoStopTheWorld(StopTheWorldCallback callback, void *argument) { + // Currently, on Emscripten this does nothing and just calls the callback. + // This works fine on a single-threaded environment. StopTheWorld(callback, argument); } diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc index bb8ebf9527bba..a9bb52906b12f 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_emscripten.cc @@ -115,6 +115,8 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, } void StopTheWorld(StopTheWorldCallback callback, void *argument) { + // TODO: have some workable alternative, since we can't just fork and suspend + // the parent process. This does not matter when single thread. callback(SuspendedThreadsList(), argument); }