From 8cf36469c4ce9169f31214832f148688b0cf0eb6 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 12 Jan 2022 17:07:04 -0800 Subject: [PATCH 01/13] start [ci skip] --- tests/test_core.py | 2 +- tools/file_packager.py | 41 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index cbf925899ad57..58d1d686b32db 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8953,7 +8953,7 @@ def setUp(self): simd2 = make_run('simd2', emcc_args=['-O2', '-msimd128']) bulkmem2 = make_run('bulkmem2', emcc_args=['-O2', '-mbulk-memory']) -wasmfs = make_run('wasmfs', emcc_args=['-s', 'WASMFS']) +wasmfs = make_run('wasmfs', emcc_args=['-s', 'WASMFS', '--profiling']) # SAFE_HEAP/STACK_OVERFLOW_CHECK core2s = make_run('core2s', emcc_args=['-O2'], settings={'SAFE_HEAP': 1}) diff --git a/tools/file_packager.py b/tools/file_packager.py index da56bede9adfa..013c09c1782a6 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -21,7 +21,7 @@ Usage: - file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] + file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--wasmfs] --preload , --embed See emcc --help for more details on those options. @@ -51,6 +51,11 @@ --no-node Whether to support Node.js. By default we do, which emits some extra code. + --wasmfs-c Whether to emit C code for wasmfs. This only supports embedding + (as it literally embeds the data in the C). If you prefer + preloading, you can use that normally and wasmfs will interact + with the JS normally. + Notes: * The file packager generates unix-style file paths. So if you are on windows and a file is accessed at @@ -185,6 +190,7 @@ def main(): lz4 = False use_preload_plugins = False support_node = True + wasmfs_c = False for arg in sys.argv[2:]: if arg == '--preload': @@ -218,6 +224,9 @@ def main(): elif arg == '--no-node': support_node = False leading = '' + elif arg == '--wasmfs-c': + wasmfs_c = True + leading = '' elif arg.startswith('--js-output'): jsoutput = arg.split('=', 1)[1] if '=' in arg else None leading = '' @@ -468,6 +477,8 @@ def was_seen(name): } ''') + c_output = '' + counter = 0 for file_ in data_files: filename = file_['dstpath'] @@ -475,13 +486,30 @@ def was_seen(name): basename = os.path.basename(filename) if file_['mode'] == 'embed': # Embed - data = base64_encode(utils.read_binary(file_['srcpath'])) - code += '''var fileData%d = '%s';\n''' % (counter, data) - code += ('''Module['FS_createDataFile']('%s', '%s', decodeBase64(fileData%d), true, true, false);\n''' - % (dirname, basename, counter)) + binary = utils.read_binary(file_['srcpath']) + if not wasmfs_c: + # JS output + data = base64_encode(binary) + code += '''var fileData%d = '%s';\n''' % (counter, data) + code += ('''Module['FS_createDataFile']('%s', '%s', decodeBase64(fileData%d), true, true, false);\n''' + % (dirname, basename, counter)) + else: + # C output. + # convert the binary data into a C escaped string, \xAB for hex code AB + def escape_for_c(char): + if char < 16: + # add a 0 to keep the output in two bytes + return '\\x0' + hex(char)[-1:] + return '\\x' + hex(char)[-2:] + data = ''.join([escape_for_c(char) for char in binary]) + c_output += '''static const char fileData%d = "%s";\n''' % (counter, data) + c_output += ('''wasmfs_create_data_file("%s", "%s", fileData%d);\n''' + % (dirname, basename, counter)) counter += 1 elif file_['mode'] == 'preload': # Preload + assert not wasmfs_c, 'wasmfs-c mode only supports embedding' + counter += 1 metadata_el = { @@ -920,6 +948,9 @@ def was_seen(name): })(); ''' % _metadata_template + if wasmfs_c: + ret = c_output + if force or len(data_files): if jsoutput is None: print(ret) From b7117e9ba7d252aa02486ad2acad7fa7a18abf56 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 15:43:53 -0800 Subject: [PATCH 02/13] work [ci skip] --- emcc.py | 15 +++++++++++---- system/lib/wasmfs/file.h | 2 ++ system/lib/wasmfs/wasmfs.cpp | 19 +++++++++++++------ system/lib/wasmfs/wasmfs.h | 6 +++--- tools/file_packager.py | 23 ++++++++++++++++++----- 5 files changed, 47 insertions(+), 18 deletions(-) diff --git a/emcc.py b/emcc.py index 08f355c2ca975..7e6bb7c48809a 100755 --- a/emcc.py +++ b/emcc.py @@ -2715,13 +2715,13 @@ def phase_source_transforms(options, target): if len(options.preload_files) or len(options.embed_files): logger.debug('setting up files') file_args = ['--from-emcc', '--export-name=' + settings.EXPORT_NAME] - if len(options.preload_files): + if options.preload_files: file_args.append('--preload') file_args += options.preload_files - if len(options.embed_files): + if options.embed_files: file_args.append('--embed') file_args += options.embed_files - if len(options.exclude_files): + if options.exclude_files: file_args.append('--exclude') file_args += options.exclude_files if options.use_preload_cache: @@ -2732,8 +2732,15 @@ def phase_source_transforms(options, target): file_args.append('--use-preload-plugins') if not settings.ENVIRONMENT_MAY_BE_NODE: file_args.append('--no-node') + wasmfs_c = settings.WASMFS and options.embed_files + if wasmfs_c: + file_args += ['--wasmfs-c'] file_code = shared.check_call([shared.FILE_PACKAGER, shared.replace_suffix(target, '.data')] + file_args, stdout=PIPE).stdout - options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code) + if wasmfs_c: + print(file_code) + 1/0 + else: + options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code) # Apply pre and postjs files if final_js and (options.pre_js or options.post_js): diff --git a/system/lib/wasmfs/file.h b/system/lib/wasmfs/file.h index e218e5c544db3..0bd40fc6bf8cd 100644 --- a/system/lib/wasmfs/file.h +++ b/system/lib/wasmfs/file.h @@ -130,6 +130,8 @@ class File : public std::enable_shared_from_this { std::weak_ptr parent; // This specifies which backend a file is associated with. + // TODO: Should this be a shared_ptr? Or do we assume backends are never + // deallocated? backend_t backend; }; diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index 75ce1bc29180e..eb1070a6d08a9 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -53,12 +53,8 @@ std::shared_ptr WasmFS::initRootDirectory() { return rootDirectory; } -// Initialize files specified by the --preload-file option. -// Set up directories and files in wasmFS$preloadedDirs and -// wasmFS$preloadedFiles from JS. This function will be called before any file -// operation to ensure any preloaded files are eagerly available for use. -void WasmFS::preloadFiles() { - // Debug builds only: add check to ensure preloadFiles() is called once. +void WasmFS::loadInitialFiles() { + // Debug builds only: add check to ensure loadInitialFiles() is called once. #ifndef NDEBUG static std::atomic timesCalled; timesCalled++; @@ -71,6 +67,17 @@ void WasmFS::preloadFiles() { // Ensure that files are preloaded from the main thread. assert(emscripten_is_main_runtime_thread()); + // First, handle embedded files, if there are any. If there are, a special + // symbol exists that we call to do that. + __attribute__((__weak__)) extern "C" void __wasmfs_load_embedded(); + if (__wasmfs_load_embedded) { + __wasmfs_load_embedded(); + } + + // Handle preloaded files. + // Set up directories and files in wasmFS$preloadedDirs and + // wasmFS$preloadedFiles from JS. This function will be called before any file + // operation to ensure any preloaded files are eagerly available for use. auto numFiles = _wasmfs_get_num_preloaded_files(); auto numDirs = _wasmfs_get_num_preloaded_dirs(); diff --git a/system/lib/wasmfs/wasmfs.h b/system/lib/wasmfs/wasmfs.h index cd9d2cd6bc985..c9e97a9cfa95b 100644 --- a/system/lib/wasmfs/wasmfs.h +++ b/system/lib/wasmfs/wasmfs.h @@ -33,15 +33,15 @@ class WasmFS { // dev/stderr. Refers to the same std streams in the open file table. std::shared_ptr initRootDirectory(); - // Initialize files specified by --preload-file option. - void preloadFiles(); + // Initialize files specified by --preload-file and --embed-file options. + void loadInitialFiles(); public: // Files will be preloaded in this constructor. // This global constructor has init_priority 100. Please see wasmfs.cpp. // The current working directory is initialized to the root directory. WasmFS() : rootDirectory(initRootDirectory()), cwd(rootDirectory) { - preloadFiles(); + loadInitialFiles(); } // This get method returns a locked file table. diff --git a/tools/file_packager.py b/tools/file_packager.py index 013c09c1782a6..1805d9ac59d74 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -21,7 +21,7 @@ Usage: - file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--wasmfs] + file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--wasmfs-c] --preload , --embed See emcc --help for more details on those options. @@ -502,9 +502,13 @@ def escape_for_c(char): return '\\x0' + hex(char)[-1:] return '\\x' + hex(char)[-2:] data = ''.join([escape_for_c(char) for char in binary]) - c_output += '''static const char fileData%d = "%s";\n''' % (counter, data) - c_output += ('''wasmfs_create_data_file("%s", "%s", fileData%d);\n''' - % (dirname, basename, counter)) + c_output += f'''static const char fileData{counter} = "{data}";\n''' + c_output += f''' +FILE* file{counter} = fopen("{dirname}" "{basename}", "wb"); +fwrite(fileData{counter}, 1, {len(binary)}, file{counter}); +fclose(file{counter}); +''' + counter += 1 elif file_['mode'] == 'preload': # Preload @@ -949,7 +953,16 @@ def escape_for_c(char): ''' % _metadata_template if wasmfs_c: - ret = c_output + ret = r''' +#include + +extern "C" void __wasmfs_load_embedded() { + +%s + +} +''' % c_output + if force or len(data_files): if jsoutput is None: From 77fb35177dc67e9065359216109ab1d232f2c907 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 15:48:02 -0800 Subject: [PATCH 03/13] [ci skip] --- emcc.py | 61 +++++++++++++++++++----------------- system/lib/wasmfs/wasmfs.cpp | 8 +++-- tools/file_packager.py | 2 +- 3 files changed, 39 insertions(+), 32 deletions(-) diff --git a/emcc.py b/emcc.py index 7e6bb7c48809a..38278626dfc4d 100755 --- a/emcc.py +++ b/emcc.py @@ -2707,40 +2707,45 @@ def phase_emscript(options, in_wasm, wasm_target, memfile): save_intermediate('original') +def package_files(options, target): + logger.debug('setting up files') + file_args = ['--from-emcc', '--export-name=' + settings.EXPORT_NAME] + if options.preload_files: + file_args.append('--preload') + file_args += options.preload_files + if options.embed_files: + file_args.append('--embed') + file_args += options.embed_files + if options.exclude_files: + file_args.append('--exclude') + file_args += options.exclude_files + if options.use_preload_cache: + file_args.append('--use-preload-cache') + if settings.LZ4: + file_args.append('--lz4') + if options.use_preload_plugins: + file_args.append('--use-preload-plugins') + if not settings.ENVIRONMENT_MAY_BE_NODE: + file_args.append('--no-node') + wasmfs_c = settings.WASMFS and options.embed_files + if wasmfs_c: + file_args += ['--wasmfs-c'] + file_code = shared.check_call([shared.FILE_PACKAGER, shared.replace_suffix(target, '.data')] + file_args, stdout=PIPE).stdout + if wasmfs_c: + print(file_code) + 1/0 + else: + options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code) + return file_code + + @ToolchainProfiler.profile_block('source transforms') def phase_source_transforms(options, target): global final_js # Embed and preload files if len(options.preload_files) or len(options.embed_files): - logger.debug('setting up files') - file_args = ['--from-emcc', '--export-name=' + settings.EXPORT_NAME] - if options.preload_files: - file_args.append('--preload') - file_args += options.preload_files - if options.embed_files: - file_args.append('--embed') - file_args += options.embed_files - if options.exclude_files: - file_args.append('--exclude') - file_args += options.exclude_files - if options.use_preload_cache: - file_args.append('--use-preload-cache') - if settings.LZ4: - file_args.append('--lz4') - if options.use_preload_plugins: - file_args.append('--use-preload-plugins') - if not settings.ENVIRONMENT_MAY_BE_NODE: - file_args.append('--no-node') - wasmfs_c = settings.WASMFS and options.embed_files - if wasmfs_c: - file_args += ['--wasmfs-c'] - file_code = shared.check_call([shared.FILE_PACKAGER, shared.replace_suffix(target, '.data')] + file_args, stdout=PIPE).stdout - if wasmfs_c: - print(file_code) - 1/0 - else: - options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code) + file_code = package_files(options, target) # Apply pre and postjs files if final_js and (options.pre_js or options.post_js): diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index eb1070a6d08a9..3c134839ae1f4 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -53,6 +53,10 @@ std::shared_ptr WasmFS::initRootDirectory() { return rootDirectory; } +// If files are embedded in the program, then this symbol is defined. We will +// call it and it will set those files up. +__attribute__((__weak__)) extern "C" void __wasmfs_load_embedded(); + void WasmFS::loadInitialFiles() { // Debug builds only: add check to ensure loadInitialFiles() is called once. #ifndef NDEBUG @@ -67,9 +71,7 @@ void WasmFS::loadInitialFiles() { // Ensure that files are preloaded from the main thread. assert(emscripten_is_main_runtime_thread()); - // First, handle embedded files, if there are any. If there are, a special - // symbol exists that we call to do that. - __attribute__((__weak__)) extern "C" void __wasmfs_load_embedded(); + // First, handle embedded files, if there are any. if (__wasmfs_load_embedded) { __wasmfs_load_embedded(); } diff --git a/tools/file_packager.py b/tools/file_packager.py index 1805d9ac59d74..21cd88ba8c8e6 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -956,7 +956,7 @@ def escape_for_c(char): ret = r''' #include -extern "C" void __wasmfs_load_embedded() { +void __wasmfs_load_embedded() { %s From 5c97e00214a08b420fd7e13acac8a1ad95a0bae0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 16:00:49 -0800 Subject: [PATCH 04/13] acks [ci skip] --- emcc.py | 16 +++++++++++----- tools/file_packager.py | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/emcc.py b/emcc.py index 38278626dfc4d..2f2b1d4d138ad 100755 --- a/emcc.py +++ b/emcc.py @@ -1093,7 +1093,7 @@ def run(args): # For internal consistency, ensure we don't attempt or read or write any link time # settings until we reach the linking phase. - settings.limit_settings(COMPILE_TIME_SETTINGS) + settings.limit_settings(list(COMPILE_TIME_SETTINGS) + ['WASMFS', 'EXPORT_NAME', 'LZ4', 'ENVIRONMENT_MAY_BE_NODE']) newargs, input_files = phase_setup(options, state, newargs, settings_map) @@ -1416,6 +1416,14 @@ def default_setting(name, new_default): if settings.EXCEPTION_HANDLING: exit_with_error('Wasm SjLj is not supported with Wasm exceptions yet') + if settings.WASMFS and options.embed_files: + # wasmfs file embedding is done via emitting C code that contains the data + # and code to set them up. we add that as another input + wasmfs_temp_file = shared.configuration.get_temp_files().get(suffix='.c').name + with open(wasmfs_temp_file, 'w') as f: + f.write(package_files(options, 'waka')) + input_files.append((len(input_files), wasmfs_temp_file)) + return (newargs, input_files) @@ -2732,11 +2740,9 @@ def package_files(options, target): file_args += ['--wasmfs-c'] file_code = shared.check_call([shared.FILE_PACKAGER, shared.replace_suffix(target, '.data')] + file_args, stdout=PIPE).stdout if wasmfs_c: - print(file_code) - 1/0 + return file_code else: options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code) - return file_code @ToolchainProfiler.profile_block('source transforms') @@ -2745,7 +2751,7 @@ def phase_source_transforms(options, target): # Embed and preload files if len(options.preload_files) or len(options.embed_files): - file_code = package_files(options, target) + package_files(options, target) # Apply pre and postjs files if final_js and (options.pre_js or options.post_js): diff --git a/tools/file_packager.py b/tools/file_packager.py index 21cd88ba8c8e6..139c19c7ea4cc 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -502,7 +502,7 @@ def escape_for_c(char): return '\\x0' + hex(char)[-1:] return '\\x' + hex(char)[-2:] data = ''.join([escape_for_c(char) for char in binary]) - c_output += f'''static const char fileData{counter} = "{data}";\n''' + c_output += f'''static const char fileData{counter}[] = "{data}";\n''' c_output += f''' FILE* file{counter} = fopen("{dirname}" "{basename}", "wb"); fwrite(fileData{counter}, 1, {len(binary)}, file{counter}); From dfd4c721864bbde5c242abfeec87c8917cb4b4bb Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 16:05:51 -0800 Subject: [PATCH 05/13] why not link [ci skip] --- tools/file_packager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/file_packager.py b/tools/file_packager.py index 139c19c7ea4cc..84c79f93de8d7 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -954,9 +954,10 @@ def escape_for_c(char): if wasmfs_c: ret = r''' +#include #include -void __wasmfs_load_embedded() { +EMSCRIPTEN_KEEPALIVE void __wasmfs_load_embedded() { %s From 9ea79c9bf8abfc5ec5bbcc251360d6a94a9e0b88 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 16:24:07 -0800 Subject: [PATCH 06/13] works [ci skip] --- emcc.py | 2 +- system/lib/wasmfs/wasmfs.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/emcc.py b/emcc.py index 2f2b1d4d138ad..547b4aa20cb18 100755 --- a/emcc.py +++ b/emcc.py @@ -1419,7 +1419,7 @@ def default_setting(name, new_default): if settings.WASMFS and options.embed_files: # wasmfs file embedding is done via emitting C code that contains the data # and code to set them up. we add that as another input - wasmfs_temp_file = shared.configuration.get_temp_files().get(suffix='.c').name + wasmfs_temp_file = '/tmp/emscripten_temp/waka.c' # shared.configuration.get_temp_files().get(suffix='.c').name with open(wasmfs_temp_file, 'w') as f: f.write(package_files(options, 'waka')) input_files.append((len(input_files), wasmfs_temp_file)) diff --git a/system/lib/wasmfs/wasmfs.cpp b/system/lib/wasmfs/wasmfs.cpp index 3c134839ae1f4..fd2c227e399c9 100644 --- a/system/lib/wasmfs/wasmfs.cpp +++ b/system/lib/wasmfs/wasmfs.cpp @@ -55,7 +55,8 @@ std::shared_ptr WasmFS::initRootDirectory() { // If files are embedded in the program, then this symbol is defined. We will // call it and it will set those files up. -__attribute__((__weak__)) extern "C" void __wasmfs_load_embedded(); +__attribute__((__weak__)) +extern "C" void __wasmfs_load_embedded(); void WasmFS::loadInitialFiles() { // Debug builds only: add check to ensure loadInitialFiles() is called once. From 8f44c3aa14ee8ad3251da9897ca7bff05012d862 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 13 Jan 2022 16:33:34 -0800 Subject: [PATCH 07/13] fixes [ci skip] --- emcc.py | 1 + tools/file_packager.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/emcc.py b/emcc.py index 547b4aa20cb18..0176a9e0c514b 100755 --- a/emcc.py +++ b/emcc.py @@ -1420,6 +1420,7 @@ def default_setting(name, new_default): # wasmfs file embedding is done via emitting C code that contains the data # and code to set them up. we add that as another input wasmfs_temp_file = '/tmp/emscripten_temp/waka.c' # shared.configuration.get_temp_files().get(suffix='.c').name + print(package_files(options, 'waka')) with open(wasmfs_temp_file, 'w') as f: f.write(package_files(options, 'waka')) input_files.append((len(input_files), wasmfs_temp_file)) diff --git a/tools/file_packager.py b/tools/file_packager.py index 84c79f93de8d7..1d2b79e9166d0 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -389,6 +389,8 @@ def was_seen(name): metadata = {'files': []} + c_output = '' + # Set up folders partial_dirs = [] for file_ in data_files: @@ -401,6 +403,7 @@ def was_seen(name): if partial not in partial_dirs: code += ('''Module['FS_createPath'](%s, %s, true, true);\n''' % (json.dumps('/' + '/'.join(parts[:i])), json.dumps(parts[i]))) + c_output += f'mkdir("{partial}", 0700);\n' partial_dirs.append(partial) if has_preloaded: @@ -477,8 +480,6 @@ def was_seen(name): } ''') - c_output = '' - counter = 0 for file_ in data_files: filename = file_['dstpath'] @@ -502,9 +503,10 @@ def escape_for_c(char): return '\\x0' + hex(char)[-1:] return '\\x' + hex(char)[-2:] data = ''.join([escape_for_c(char) for char in binary]) + # directories... :( make them c_output += f'''static const char fileData{counter}[] = "{data}";\n''' c_output += f''' -FILE* file{counter} = fopen("{dirname}" "{basename}", "wb"); +FILE* file{counter} = fopen("{dirname}" "/" "{basename}", "wb"); fwrite(fileData{counter}, 1, {len(binary)}, file{counter}); fclose(file{counter}); ''' @@ -956,6 +958,7 @@ def escape_for_c(char): ret = r''' #include #include +#include EMSCRIPTEN_KEEPALIVE void __wasmfs_load_embedded() { From 43af496c364170b76d54907abe49d16311afa2f7 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 10:31:36 -0800 Subject: [PATCH 08/13] better --- emcc.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/emcc.py b/emcc.py index 0176a9e0c514b..60f56a2a91d09 100755 --- a/emcc.py +++ b/emcc.py @@ -1135,7 +1135,7 @@ def run(args): logger.debug('stopping after linking to object file') return 0 - phase_calculate_system_libraries(state, linker_arguments, linker_inputs, newargs) + phase_calculate_system_libraries(state, linker_arguments, linker_inputs, newargs, options) phase_link(linker_arguments, wasm_target) @@ -1416,15 +1416,6 @@ def default_setting(name, new_default): if settings.EXCEPTION_HANDLING: exit_with_error('Wasm SjLj is not supported with Wasm exceptions yet') - if settings.WASMFS and options.embed_files: - # wasmfs file embedding is done via emitting C code that contains the data - # and code to set them up. we add that as another input - wasmfs_temp_file = '/tmp/emscripten_temp/waka.c' # shared.configuration.get_temp_files().get(suffix='.c').name - print(package_files(options, 'waka')) - with open(wasmfs_temp_file, 'w') as f: - f.write(package_files(options, 'waka')) - input_files.append((len(input_files), wasmfs_temp_file)) - return (newargs, input_files) @@ -2629,7 +2620,7 @@ def compile_source_file(i, input_file): @ToolchainProfiler.profile_block('calculate system libraries') -def phase_calculate_system_libraries(state, linker_arguments, linker_inputs, newargs): +def phase_calculate_system_libraries(state, linker_arguments, linker_inputs, newargs, options): extra_files_to_link = [] # link in ports and system libraries, if necessary if not settings.SIDE_MODULE: @@ -2639,6 +2630,20 @@ def phase_calculate_system_libraries(state, linker_arguments, linker_inputs, new extra_files_to_link += system_libs.calculate(all_linker_inputs, newargs, forced=state.forced_stdlibs) linker_arguments.extend(extra_files_to_link) + if settings.WASMFS and options.embed_files: + # wasmfs file embedding is done via emitting C code that contains the data + # and code to set them up. we add that as another input, like a system + # library, that we compile on the fly here + temp_files = shared.configuration.get_temp_files() + temp_c = temp_files.get(suffix='.c').name + temp_o = unsuffixed(temp_c) + '.o' + temp_files.note(temp_o) + #print(package_files(options, 'waka')) + with open(temp_c, 'w') as f: + f.write(package_files(options, 'waka')) + shared.check_call([shared.CLANG_CC, temp_c, '-o', temp_o, '-c'] + get_cflags([])) + linker_arguments.append(temp_o) + @ToolchainProfiler.profile_block('link') def phase_link(linker_arguments, wasm_target): From 299a4fedbcd93f5c64c81de3ec0f82371e6202ba Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 10:34:14 -0800 Subject: [PATCH 09/13] test --- emcc.py | 2 +- tests/test_core.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/emcc.py b/emcc.py index 60f56a2a91d09..313a6b32fcd60 100755 --- a/emcc.py +++ b/emcc.py @@ -1093,7 +1093,7 @@ def run(args): # For internal consistency, ensure we don't attempt or read or write any link time # settings until we reach the linking phase. - settings.limit_settings(list(COMPILE_TIME_SETTINGS) + ['WASMFS', 'EXPORT_NAME', 'LZ4', 'ENVIRONMENT_MAY_BE_NODE']) + settings.limit_settings(COMPILE_TIME_SETTINGS) newargs, input_files = phase_setup(options, state, newargs, settings_map) diff --git a/tests/test_core.py b/tests/test_core.py index 58d1d686b32db..9aa985991758e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -528,6 +528,7 @@ def test_wasm32_unknown_emscripten(self): # No other configuration is supported, so always run this. self.do_runf(test_file('wasm32-unknown-emscripten.c'), '') + @also_with_wasmfs # tests file embedding def test_cube2md5(self): self.emcc_args += ['--embed-file', 'cube2md5.txt'] shutil.copyfile(test_file('cube2md5.txt'), 'cube2md5.txt') From 1ff958fd27dd506ae2e4a06e227e96db44d021e7 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 10:36:32 -0800 Subject: [PATCH 10/13] cleanup --- emcc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/emcc.py b/emcc.py index 313a6b32fcd60..bfe934e51acaf 100755 --- a/emcc.py +++ b/emcc.py @@ -2638,7 +2638,6 @@ def phase_calculate_system_libraries(state, linker_arguments, linker_inputs, new temp_c = temp_files.get(suffix='.c').name temp_o = unsuffixed(temp_c) + '.o' temp_files.note(temp_o) - #print(package_files(options, 'waka')) with open(temp_c, 'w') as f: f.write(package_files(options, 'waka')) shared.check_call([shared.CLANG_CC, temp_c, '-o', temp_o, '-c'] + get_cflags([])) From 8ecce7ba503fab4f51a066f5ae2c550861462038 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 10:41:58 -0800 Subject: [PATCH 11/13] fix hack --- emcc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/emcc.py b/emcc.py index bfe934e51acaf..1039ee30bb118 100755 --- a/emcc.py +++ b/emcc.py @@ -2639,7 +2639,7 @@ def phase_calculate_system_libraries(state, linker_arguments, linker_inputs, new temp_o = unsuffixed(temp_c) + '.o' temp_files.note(temp_o) with open(temp_c, 'w') as f: - f.write(package_files(options, 'waka')) + f.write(package_files(options)) shared.check_call([shared.CLANG_CC, temp_c, '-o', temp_o, '-c'] + get_cflags([])) linker_arguments.append(temp_o) @@ -2720,7 +2720,7 @@ def phase_emscript(options, in_wasm, wasm_target, memfile): save_intermediate('original') -def package_files(options, target): +def package_files(options, target='default'): logger.debug('setting up files') file_args = ['--from-emcc', '--export-name=' + settings.EXPORT_NAME] if options.preload_files: From 7b65d4a1759f8b2b9f30a9cd444420bd29265967 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 10:43:01 -0800 Subject: [PATCH 12/13] remove hack --- tools/file_packager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/file_packager.py b/tools/file_packager.py index 1d2b79e9166d0..70319e52e09ad 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -956,11 +956,10 @@ def escape_for_c(char): if wasmfs_c: ret = r''' -#include #include #include -EMSCRIPTEN_KEEPALIVE void __wasmfs_load_embedded() { +void __wasmfs_load_embedded() { %s From 832b173504bc8936f8f601f4fc1bf7729e430900 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 14 Jan 2022 10:46:12 -0800 Subject: [PATCH 13/13] flake8 --- tools/file_packager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/file_packager.py b/tools/file_packager.py index 70319e52e09ad..1a7663d37b74e 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -966,7 +966,6 @@ def escape_for_c(char): } ''' % c_output - if force or len(data_files): if jsoutput is None: print(ret)