From adbf8dd7a644f00300d220f890738912a51f9402 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Sat, 15 Jan 2022 07:19:09 -0800 Subject: [PATCH] file_packager.py: Add option to embed file data in wasm binary This change not only adds the new option but uses this option whenever file_packager is used from within emcc. Hopefully we can find a way deprecate and remove the old JS embedded since that seems strictly worse in almost ever way. - Larger code size (JS base64 encoding is larger than binary) - No possiblity of zero copy, memory-backed files - Less compatible with standalone wasm / WASI --- ChangeLog.md | 4 + emcc.py | 78 +++++---- src/library_fs.js | 6 +- .../hello_world_O3_MAIN_MODULE_2.funcs | 3 +- tests/test_other.py | 10 +- tools/file_packager.py | 162 +++++++++++++++++- tools/shared.py | 8 + 7 files changed, 223 insertions(+), 48 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index ae86fc693281e..44873df201309 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -20,6 +20,10 @@ See docs/process.md for more on how version tagging works. 3.1.3 ----- +- The file packager now supports embedding files directly into wasm memory and + `emcc` now uses this mode when the `--embed-file` option is used. If you + use `file_packager` directly it is recommended that you switch to the new mode + by adding `--obj-output` to the command line. (#16050) 3.1.2 - 20/01/2022 ------------------ diff --git a/emcc.py b/emcc.py index 347c2e1be55bb..07a8cf305edeb 100755 --- a/emcc.py +++ b/emcc.py @@ -814,14 +814,7 @@ def array_contains_any_of(hay, needles): def get_clang_flags(): - return ['-target', get_llvm_target()] - - -def get_llvm_target(): - if settings.MEMORY64: - return 'wasm64-unknown-emscripten' - else: - return 'wasm32-unknown-emscripten' + return ['-target', shared.get_llvm_target()] cflags = None @@ -979,6 +972,41 @@ def get_subresource_location(path, data_uri=None): return os.path.basename(path) +@ToolchainProfiler.profile_block('package_files') +def package_files(options, target): + rtn = [] + logger.debug('setting up files') + file_args = ['--from-emcc', '--export-name=' + settings.EXPORT_NAME] + if options.preload_files: + file_args.append('--preload') + file_args += options.preload_files + if options.embed_files: + file_args.append('--embed') + file_args += options.embed_files + if options.exclude_files: + file_args.append('--exclude') + file_args += options.exclude_files + if options.use_preload_cache: + file_args.append('--use-preload-cache') + if settings.LZ4: + file_args.append('--lz4') + if options.use_preload_plugins: + file_args.append('--use-preload-plugins') + if not settings.ENVIRONMENT_MAY_BE_NODE: + file_args.append('--no-node') + if options.embed_files: + object_file = in_temp('embedded_files.o') + file_args += ['--obj-output=' + object_file] + rtn.append(object_file) + + cmd = [shared.FILE_PACKAGER, shared.replace_suffix(target, '.data')] + file_args + file_code = shared.check_call(cmd, stdout=PIPE).stdout + + options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code) + + return rtn + + run_via_emxx = False @@ -1053,7 +1081,7 @@ def run(args): return 0 if '-dumpmachine' in args: - print(get_llvm_target()) + print(shared.get_llvm_target()) return 0 if '-dumpversion' in args: # gcc's doc states "Print the compiler version [...] and don't do anything else." @@ -1129,6 +1157,10 @@ def run(args): # Link object files using wasm-ld or llvm-link (for bitcode linking) linker_arguments = phase_calculate_linker_inputs(options, state, linker_inputs) + # Embed and preload files + if len(options.preload_files) or len(options.embed_files): + linker_arguments += package_files(options, target) + if options.oformat == OFormat.OBJECT: logger.debug(f'link_to_object: {linker_arguments} -> {target}') building.link_to_object(linker_arguments, target) @@ -2699,7 +2731,7 @@ def phase_post_link(options, state, in_wasm, wasm_target, target): phase_emscript(options, in_wasm, wasm_target, memfile) - phase_source_transforms(options, target) + phase_source_transforms(options) if memfile and not settings.MINIMAL_RUNTIME: # MINIMAL_RUNTIME doesn't use `var memoryInitializer` but instead expects Module['mem'] to @@ -2730,33 +2762,9 @@ def phase_emscript(options, in_wasm, wasm_target, memfile): @ToolchainProfiler.profile_block('source transforms') -def phase_source_transforms(options, target): +def phase_source_transforms(options): global final_js - # Embed and preload files - if len(options.preload_files) or len(options.embed_files): - logger.debug('setting up files') - file_args = ['--from-emcc', '--export-name=' + settings.EXPORT_NAME] - if len(options.preload_files): - file_args.append('--preload') - file_args += options.preload_files - if len(options.embed_files): - file_args.append('--embed') - file_args += options.embed_files - if len(options.exclude_files): - file_args.append('--exclude') - file_args += options.exclude_files - if options.use_preload_cache: - file_args.append('--use-preload-cache') - if settings.LZ4: - file_args.append('--lz4') - if options.use_preload_plugins: - file_args.append('--use-preload-plugins') - if not settings.ENVIRONMENT_MAY_BE_NODE: - file_args.append('--no-node') - file_code = shared.check_call([shared.FILE_PACKAGER, shared.replace_suffix(target, '.data')] + file_args, stdout=PIPE).stdout - options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code) - # Apply pre and postjs files if final_js and (options.pre_js or options.post_js): logger.debug('applying pre/postjses') diff --git a/src/library_fs.js b/src/library_fs.js index e17511623226c..7d95dfe3cd2fe 100644 --- a/src/library_fs.js +++ b/src/library_fs.js @@ -1590,7 +1590,11 @@ FS.staticInit();` + return FS.create(path, mode); }, createDataFile: function(parent, name, data, canRead, canWrite, canOwn) { - var path = name ? PATH.join2(typeof parent === 'string' ? parent : FS.getPath(parent), name) : parent; + var path = name; + if (parent) { + parent = typeof parent === 'string' ? parent : FS.getPath(parent); + path = name ? PATH.join2(parent, name) : parent; + } var mode = FS.getMode(canRead, canWrite); var node = FS.create(path, mode); if (data) { diff --git a/tests/other/metadce/hello_world_O3_MAIN_MODULE_2.funcs b/tests/other/metadce/hello_world_O3_MAIN_MODULE_2.funcs index 2fe0b3e5c4554..dc45913607854 100644 --- a/tests/other/metadce/hello_world_O3_MAIN_MODULE_2.funcs +++ b/tests/other/metadce/hello_world_O3_MAIN_MODULE_2.funcs @@ -3,8 +3,9 @@ $__emscripten_stdout_seek $__fwritex $__stdio_write $__towrite -$__wasm_apply_global_relocs +$__wasm_apply_data_relocs $__wasm_call_ctors +$__wasm_start $dlmalloc $legalstub$dynCall_jiji $main diff --git a/tests/test_other.py b/tests/test_other.py index 7b5fccfea7b93..61420e2909f96 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -2590,7 +2590,7 @@ def test_file_packager_directory_with_single_quote(self): assert json.dumps("direc'tory") in proc.stdout def test_file_packager_mention_FORCE_FILESYSTEM(self): - MESSAGE = 'Remember to build the main file with -s FORCE_FILESYSTEM=1 so that it includes support for loading this file package' + MESSAGE = 'Remember to build the main file with `-sFORCE_FILESYSTEM` so that it includes support for loading this file package' create_file('data.txt', 'data1') # mention when running standalone err = self.run_process([FILE_PACKAGER, 'test.data', '--preload', 'data.txt'], stdout=PIPE, stderr=PIPE).stderr @@ -2608,7 +2608,11 @@ def test_file_packager_returns_error_if_target_equal_to_jsoutput(self): def test_file_packager_embed(self): create_file('data.txt', 'hello data') - self.run_process([FILE_PACKAGER, 'test.data', '--embed', 'data.txt', '--js-output=data.js']) + # Without --obj-output we issue a warning + err = self.run_process([FILE_PACKAGER, 'test.data', '--embed', 'data.txt', '--js-output=data.js'], stderr=PIPE).stderr + self.assertContained('--obj-output is recommended when using --embed', err) + + self.run_process([FILE_PACKAGER, 'test.data', '--embed', 'data.txt', '--obj-output=data.o', '--js-output=data.js']) create_file('test.c', ''' #include @@ -2623,7 +2627,7 @@ def test_file_packager_embed(self): return 0; } ''') - self.run_process([EMCC, '--pre-js=data.js', 'test.c', '-sFORCE_FILESYSTEM']) + self.run_process([EMCC, '--pre-js=data.js', 'test.c', 'data.o', '-sFORCE_FILESYSTEM']) output = self.run_js('a.out.js') self.assertContained('hello data', output) diff --git a/tools/file_packager.py b/tools/file_packager.py index 97484642ef23f..3eaaf2b1259b8 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -16,7 +16,7 @@ * If you run this yourself, separately/standalone from emcc, then the main program compiled by emcc must be built with filesystem support. You can do that with - -s FORCE_FILESYSTEM=1 (if you forget that, an unoptimized build or one with + -sFORCE_FILESYSTEM (if you forget that, an unoptimized build or one with ASSERTIONS enabled will show an error suggesting you use that flag). Usage: @@ -33,6 +33,8 @@ --js-output=FILE Writes output in FILE, if not specified, standard output is used. + --obj-output=FILE create an object file from embedded files, for direct linking into a wasm binary. + --export-name=EXPORT_NAME Use custom export name (default is `Module`) --no-force Don't create output if no valid input file is specified. @@ -68,6 +70,7 @@ import sys import uuid from subprocess import PIPE +from textwrap import dedent __scriptdir__ = os.path.dirname(os.path.abspath(__file__)) __rootdir__ = os.path.dirname(__scriptdir__) @@ -96,7 +99,9 @@ class Options: def __init__(self): self.export_name = 'Module' self.has_preloaded = False + self.has_embedded = False self.jsoutput = None + self.obj_output = None self.from_emcc = False self.force = True # If set to True, IndexedDB (IDBFS in library_idbfs.js) is used to locally @@ -194,6 +199,121 @@ def add(mode, rootpathsrc, rootpathdst): dirnames.extend(new_dirnames) +def to_asm_string(string): + """Convert a python string to string suitable for including in an + assembly file using the `.asciz` directive. + + The result will be an UTF-8 encoded string in the data section. + """ + # See MCAsmStreamer::PrintQuotedString in llvm/lib/MC/MCAsmStreamer.cpp + # And isPrint in llvm/include/llvm/ADT/StringExtras.h + + def is_print(c): + return c >= 0x20 and c <= 0x7E + + def escape(c): + if is_print(c): + return chr(c) + escape_chars = { + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', + } + if c in escape_chars: + return escape_chars[c] + # Enscode all other chars are three octal digits(!) + return '\\%s%s%s' % (oct(c >> 6), oct(c >> 3), oct(c >> 0)) + + return ''.join(escape(c) for c in string.encode('utf-8')) + + +def to_c_symbol(filename, used): + """Convert a filename (python string) to a legal C symbols, avoiding collisions.""" + def escape(c): + if c.isalnum(): + return c + else: + return '_' + c_symbol = ''.join(escape(c) for c in filename) + # Handle collisions + if c_symbol in used: + counter = 2 + while c_symbol + str(counter) in used: + counter = counter + 1 + c_symbol = c_symbol + str(counter) + used.add(c_symbol) + return c_symbol + + +def generate_object_file(data_files): + embed_files = [f for f in data_files if f.mode == 'embed'] + assert embed_files + + asm_file = shared.replace_suffix(options.obj_output, '.s') + + used = set() + for f in embed_files: + f.c_symbol_name = '__em_file_data_%s' % to_c_symbol(f.dstpath, used) + + with open(asm_file, 'w') as out: + out.write('# Emscripten embedded file data, generated by tools/file_packager.py\n') + + for f in embed_files: + if DEBUG: + err('embedding %s at %s' % (f.srcpath, f.dstpath)) + + size = os.path.getsize(f.srcpath) + name = to_asm_string(f.dstpath) + out.write(dedent(f''' + .section .rodata.{f.c_symbol_name},"",@ + + # The name of file + {f.c_symbol_name}_name: + .asciz "{name}" + .size {f.c_symbol_name}_name, {len(name)+1} + + # The size of the file followed by the content itself + {f.c_symbol_name}: + .incbin "{f.srcpath}" + .size {f.c_symbol_name}, {size} + ''')) + + out.write(dedent(''' + # A list of triples of: + # (file_name_ptr, file_data_size, file_data_ptr) + # The list in null terminate with a single 0 + .globl __emscripten_embedded_file_data + .export_name __emscripten_embedded_file_data, __emscripten_embedded_file_data + .section .rodata.__emscripten_embedded_file_data,"",@ + __emscripten_embedded_file_data: + .p2align 2 + ''')) + + for f in embed_files: + # The `.dc.a` directive gives us a pointer (address) sized entry. + # See https://sourceware.org/binutils/docs/as/Dc.html + out.write(dedent(f'''\ + .dc.a {f.c_symbol_name}_name + .int32 {os.path.getsize(f.srcpath)} + .dc.a {f.c_symbol_name} + ''')) + + ptr_size = 4 + elem_size = (2 * ptr_size) + 4 + total_size = len(embed_files) * elem_size + 4 + out.write(dedent(f'''\ + .dc.a 0 + .size __emscripten_embedded_file_data, {total_size} + ''')) + shared.check_call([shared.LLVM_MC, + '-filetype=obj', + '-triple=' + shared.get_llvm_target(), + '-o', options.obj_output, + asm_file]) + + def main(): if len(sys.argv) == 1: err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] @@ -239,6 +359,9 @@ def main(): elif arg.startswith('--js-output'): options.jsoutput = arg.split('=', 1)[1] if '=' in arg else None leading = '' + elif arg.startswith('--obj-output'): + options.obj_output = arg.split('=', 1)[1] if '=' in arg else None + leading = '' elif arg.startswith('--export-name'): if '=' in arg: options.export_name = arg.split('=', 1)[1] @@ -279,6 +402,7 @@ def main(): return 1 options.has_preloaded = any(f.mode == 'preload' for f in data_files) + options.has_embedded = any(f.mode == 'embed' for f in data_files) if options.separate_metadata: if not options.has_preloaded or not options.jsoutput: @@ -287,7 +411,7 @@ def main(): return 1 if not options.from_emcc: - err('Remember to build the main file with -s FORCE_FILESYSTEM=1 ' + err('Remember to build the main file with `-sFORCE_FILESYSTEM` ' 'so that it includes support for loading this file package') if options.jsoutput and os.path.abspath(options.jsoutput) == os.path.abspath(data_target): @@ -369,6 +493,12 @@ def was_seen(name): metadata = {'files': []} + if options.obj_output: + if not options.has_embedded: + err('--obj-output is only applicable when embedding files') + return 1 + generate_object_file(data_files) + ret = generate_js(data_target, data_files, metadata) if options.force or len(data_files): @@ -504,17 +634,33 @@ def generate_js(data_target, data_files, metadata): new DataRequest(files[i]['start'], files[i]['end'], files[i]['audio'] || 0).open('GET', files[i]['filename']); }\n''' % (create_preloaded if options.use_preload_plugins else create_data) + if options.has_embedded: + if options.obj_output: + code += '''\ + var start32 = Module['___emscripten_embedded_file_data'] >> 2; + do { + var name_addr = HEAPU32[start32++]; + var len = HEAPU32[start32++]; + var content = HEAPU32[start32++]; + var name = UTF8ToString(name_addr) + // canOwn this data in the filesystem, it is a slice of wasm memory that will never change + Module['FS_createDataFile'](undefined, name, HEAP8.subarray(content, content + len), true, true, true); + } while (HEAPU32[start32]);''' + else: + err('--obj-output is recommended when using --embed. This outputs an object file for linking directly into your application is more effecient than JS encoding') + for (counter, file_) in enumerate(data_files): filename = file_.dstpath dirname = os.path.dirname(filename) basename = os.path.basename(filename) if file_.mode == 'embed': - # Embed - data = base64_encode(utils.read_binary(file_.srcpath)) - code += " var fileData%d = '%s';\n" % (counter, data) - # canOwn this data in the filesystem (i.e. there is no need to create a copy in the FS layer). - code += (" Module['FS_createDataFile']('%s', '%s', decodeBase64(fileData%d), true, true, true);\n" - % (dirname, basename, counter)) + if not options.obj_output: + # Embed + data = base64_encode(utils.read_binary(file_.srcpath)) + code += " var fileData%d = '%s';\n" % (counter, data) + # canOwn this data in the filesystem (i.e. there is no need to create a copy in the FS layer). + code += (" Module['FS_createDataFile']('%s', '%s', decodeBase64(fileData%d), true, true, true);\n" + % (dirname, basename, counter)) elif file_.mode == 'preload': # Preload metadata_el = { diff --git a/tools/shared.py b/tools/shared.py index 3ed092c7dde7a..9c36d35d4638a 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -664,6 +664,13 @@ def do_replace(input_, pattern, replacement): return input_.replace(pattern, replacement) +def get_llvm_target(): + if settings.MEMORY64: + return 'wasm64-unknown-emscripten' + else: + return 'wasm32-unknown-emscripten' + + # ============================================================================ # End declarations. # ============================================================================ @@ -690,6 +697,7 @@ def do_replace(input_, pattern, replacement): LLVM_RANLIB = build_llvm_tool_path(exe_suffix('llvm-ranlib')) LLVM_OPT = os.path.expanduser(build_llvm_tool_path(exe_suffix('opt'))) LLVM_NM = os.path.expanduser(build_llvm_tool_path(exe_suffix('llvm-nm'))) +LLVM_MC = os.path.expanduser(build_llvm_tool_path(exe_suffix('llvm-mc'))) LLVM_INTERPRETER = os.path.expanduser(build_llvm_tool_path(exe_suffix('lli'))) LLVM_COMPILER = os.path.expanduser(build_llvm_tool_path(exe_suffix('llc'))) LLVM_DWARFDUMP = os.path.expanduser(build_llvm_tool_path(exe_suffix('llvm-dwarfdump')))