From 1d4e2464665ed71d5f05b2028f14c57d82905650 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 31 Mar 2023 16:32:50 -0400 Subject: [PATCH 001/110] [tasks] Move EmitWasmBundle out of wasm specific task --- .../EmitWasmBundleBase.cs => EmitBundleTask/EmitBundleBase.cs} | 0 .../EmitBundleObjectFiles.cs} | 0 .../EmitBundleSourceFiles.cs} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename src/tasks/{WasmAppBuilder/EmitWasmBundleBase.cs => EmitBundleTask/EmitBundleBase.cs} (100%) rename src/tasks/{WasmAppBuilder/EmitWasmBundleObjectFiles.cs => EmitBundleTask/EmitBundleObjectFiles.cs} (100%) rename src/tasks/{WasmAppBuilder/EmitWasmBundleSourceFiles.cs => EmitBundleTask/EmitBundleSourceFiles.cs} (100%) diff --git a/src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs similarity index 100% rename from src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs rename to src/tasks/EmitBundleTask/EmitBundleBase.cs diff --git a/src/tasks/WasmAppBuilder/EmitWasmBundleObjectFiles.cs b/src/tasks/EmitBundleTask/EmitBundleObjectFiles.cs similarity index 100% rename from src/tasks/WasmAppBuilder/EmitWasmBundleObjectFiles.cs rename to src/tasks/EmitBundleTask/EmitBundleObjectFiles.cs diff --git a/src/tasks/WasmAppBuilder/EmitWasmBundleSourceFiles.cs b/src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs similarity index 100% rename from src/tasks/WasmAppBuilder/EmitWasmBundleSourceFiles.cs rename to src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs From 2955cdb5725d3a0a3c658fc5401e8c49438368cb Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 31 Mar 2023 16:34:27 -0400 Subject: [PATCH 002/110] [tasks] Make EmitBundle more generic --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 8 +++----- src/tasks/EmitBundleTask/EmitBundleObjectFiles.cs | 4 +--- src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs | 8 +++----- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 7286e088a65fb0..dfd84a74b1a23c 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -11,9 +11,7 @@ using System.Threading.Tasks; using Microsoft.Build.Framework; -namespace Microsoft.WebAssembly.Build.Tasks; - -public abstract class EmitWasmBundleBase : Microsoft.Build.Utilities.Task, ICancelableTask +public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelableTask { private CancellationTokenSource BuildTaskCancelled { get; } = new(); @@ -65,7 +63,7 @@ public override bool Execute() return (registeredName, symbolName); }).ToList(); - Log.LogMessage(MessageImportance.Low, "Bundling {numFiles} files for {bundleName}", files.Count, BundleName); + Log.LogMessage(MessageImportance.Low, "Bundling {0} files for {1}", files.Count, BundleName); if (remainingDestinationFilesToBundle.Length > 0) { @@ -109,7 +107,7 @@ public override bool Execute() return Emit(BundleFile, (inputStream) => { using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); - GenerateRegisteredBundledObjects($"mono_wasm_register_{BundleName}_bundle", RegistrationCallbackFunctionName, files, outputUtf8Writer); + GenerateRegisteredBundledObjects($"mono_register_{BundleName}_bundle", RegistrationCallbackFunctionName, files, outputUtf8Writer); }) && !Log.HasLoggedErrors; } diff --git a/src/tasks/EmitBundleTask/EmitBundleObjectFiles.cs b/src/tasks/EmitBundleTask/EmitBundleObjectFiles.cs index 48c1f509768cd9..efdf11a23ec603 100644 --- a/src/tasks/EmitBundleTask/EmitBundleObjectFiles.cs +++ b/src/tasks/EmitBundleTask/EmitBundleObjectFiles.cs @@ -6,9 +6,7 @@ using System.IO; using Microsoft.Build.Framework; -namespace Microsoft.WebAssembly.Build.Tasks; - -public class EmitWasmBundleObjectFiles : EmitWasmBundleBase +public class EmitBundleObjectFiles : EmitBundleBase { [Required] public string ClangExecutable { get; set; } = default!; diff --git a/src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs b/src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs index b57a8f4627a82a..5e256c4eb66354 100644 --- a/src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs +++ b/src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs @@ -5,12 +5,10 @@ using System.IO; using Microsoft.Build.Framework; -namespace Microsoft.WebAssembly.Build.Tasks; - -// It would be ideal that this Task would always produce object files as EmitWasmBundleObjectFiles does. -// EmitWasmBundleObjectFiles could do it with clang by streaming code directly to clang input stream. +// It would be ideal that this Task would always produce object files as EmitBundleObjectFiles does. +// EmitBundleObjectFiles could do it with clang by streaming code directly to clang input stream. // For emcc it's not possible, so we need to write the code to disk first and then compile it in MSBuild. -public class EmitWasmBundleSourceFiles : EmitWasmBundleBase +public class EmitBundleSourceFiles : EmitBundleBase { public override bool Emit(string destinationFile, Action inputProvider) { From 2d25efc1ab6f8d4e0bcc5c4a01f7698ff857b693 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 31 Mar 2023 16:38:02 -0400 Subject: [PATCH 003/110] [tasks] Generate EmitBundle task and replace invocations of old task --- Directory.Build.props | 2 ++ src/mono/wasi/build/WasiApp.Native.targets | 6 +++--- src/mono/wasi/runtime/driver.c | 8 ++++---- src/mono/wasi/wasi.proj | 6 +++--- src/mono/wasm/runtime/driver.c | 4 ++-- src/mono/wasm/wasm.proj | 8 ++++---- src/tasks/EmitBundleTask/EmitBundle.csproj | 21 +++++++++++++++++++++ 7 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 src/tasks/EmitBundleTask/EmitBundle.csproj diff --git a/Directory.Build.props b/Directory.Build.props index 7eb57c38fc8a22..74d5ab67abc893 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -152,6 +152,7 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'LibraryBuilder', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoTargetsTasks', 'Debug', '$(NetCoreAppToolCurrent)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'EmitBundle', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'TestExclusionListTasks', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetFrameworkToolCurrent)', 'installer.tasks.dll')) @@ -166,6 +167,7 @@ $([MSBuild]::NormalizePath('$(LibraryBuilderDir)', 'LibraryBuilder.dll')) $([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll')) $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll')) + $([MSBuild]::NormalizePath('$(EmitBundleDir)', 'EmitBundle.dll')) $([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(RuntimeConfiguration)')) $(CoreCLRToolPath) diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 03a62b781b7399..3a983159f3c34b 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -341,9 +341,9 @@ - + - <_WasmAssembliesBundleObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_assemblies.o @@ -357,7 +357,7 @@ - - + <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) @@ -86,14 +86,14 @@ - - + diff --git a/src/mono/wasm/runtime/driver.c b/src/mono/wasm/runtime/driver.c index 0dc84315a929d9..77616f3f57a64b 100644 --- a/src/mono/wasm/runtime/driver.c +++ b/src/mono/wasm/runtime/driver.c @@ -53,7 +53,7 @@ int monoeg_g_setenv(const char *variable, const char *value, int overwrite); int32_t mini_parse_debug_option (const char *option); char *mono_method_get_full_name (MonoMethod *method); -extern void mono_wasm_register_timezones_bundle(); +extern void mono_register_timezones_bundle(); extern void mono_wasm_set_entrypoint_breakpoint (const char* assembly_name, int method_token); static void mono_wasm_init_finalizer_thread (void); @@ -512,7 +512,7 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mini_parse_debug_option ("top-runtime-invoke-unhandled"); - mono_wasm_register_timezones_bundle(); + mono_register_timezones_bundle(); mono_dl_fallback_register (wasm_dl_load, wasm_dl_symbol, NULL, NULL); mono_wasm_install_get_native_to_interp_tramp (get_native_to_interp); diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index 9755e405afed42..826e014e77aea9 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -80,7 +80,7 @@ - + <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) @@ -104,13 +104,13 @@ - - + /> + <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), %(_WasmBundleTimezonesWithHashes.DestinationFile)).Replace('\','/'))" /> <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), $(_WasmTimezonesBundleSourceFile)).Replace('\','/'))" /> diff --git a/src/tasks/EmitBundleTask/EmitBundle.csproj b/src/tasks/EmitBundleTask/EmitBundle.csproj new file mode 100644 index 00000000000000..34b8fd649bf896 --- /dev/null +++ b/src/tasks/EmitBundleTask/EmitBundle.csproj @@ -0,0 +1,21 @@ + + + $(TargetFrameworkForNETCoreTasks) + enable + $(NoWarn),CA1050,CA1850 + + + + + + + + + + + + + + + From d8d97e84d7b1102024a80311d9489dd420816ba2 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 31 Mar 2023 16:43:05 -0400 Subject: [PATCH 004/110] Extend preloading assemblies with mono_assembly_open --- src/tasks/LibraryBuilder/LibraryBuilder.cs | 2 +- .../LibraryBuilder/Templates/library-builder.h | 1 + .../Templates/preloaded-assemblies.c | 16 +++++++++++++++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index dd3a51324f5a8e..5cbd2ee0adc7c3 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -179,7 +179,7 @@ private void GatherAotSourcesObjects(StringBuilder aotSources, StringBuilder aot if (symbolsAdded > 0) { - exportedAssemblies.Add(Path.GetFileNameWithoutExtension(compiledAssembly.Path)); + exportedAssemblies.Add(Path.GetFileName(compiledAssembly.Path)); } } } diff --git a/src/tasks/LibraryBuilder/Templates/library-builder.h b/src/tasks/LibraryBuilder/Templates/library-builder.h index c0dee67be95adc..a3fdb705caefee 100644 --- a/src/tasks/LibraryBuilder/Templates/library-builder.h +++ b/src/tasks/LibraryBuilder/Templates/library-builder.h @@ -5,6 +5,7 @@ #define __MONO_LIBRARY_BUILDER_H__ #include +#include #if defined(HOST_ANDROID) diff --git a/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c b/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c index 71be26d870dc0b..4f949ff788b6c7 100644 --- a/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c +++ b/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c @@ -9,7 +9,21 @@ static void preload_assembly (const char* filename) { - MonoAssembly *assembly = mono_assembly_load_with_partial_name (filename, NULL); + MonoAssembly *assembly = mono_assembly_open (filename, NULL); + if (assembly) + return; + + int len = strlen (filename); + char *filename_without_extension = strdup (filename); + if (!filename_without_extension) + LOG_ERROR ("Out of memory.\n"); + + if (!strcasecmp (".dll", &filename [len - 4])) + *(filename_without_extension + len - 4) = '\0'; + + assembly = mono_assembly_load_with_partial_name (filename_without_extension, NULL); + + free (filename_without_extension); if (!assembly) LOG_ERROR ("Could not open assembly '%s'.\n", filename); } From eb5e39636d45be010ae451f501e42cc8faa3ca2a Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 31 Mar 2023 17:55:26 -0400 Subject: [PATCH 005/110] [LibraryBuilder] Integrate bundling --- src/mono/mono/mini/aot-runtime.c | 36 +++++++++++++++++++ src/mono/mono/mini/aot-runtime.h | 2 ++ .../msbuild/common/LibraryBuilder.targets | 29 +++++++++++++++ src/tasks/LibraryBuilder/LibraryBuilder.cs | 10 ++++++ src/tasks/LibraryBuilder/Templates/autoinit.c | 7 +++- .../Templates/library-builder.h | 6 +++- 6 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/mono/mono/mini/aot-runtime.c b/src/mono/mono/mini/aot-runtime.c index 46e0aff893914f..d0323ea94a1fe3 100644 --- a/src/mono/mono/mini/aot-runtime.c +++ b/src/mono/mono/mini/aot-runtime.c @@ -2413,6 +2413,42 @@ mono_aot_register_module (gpointer *aot_info) mono_aot_unlock (); } +typedef struct BundleAssembly_ BundleAssembly; + +struct BundleAssembly_ { + MonoBundledAssembly assembly; + BundleAssembly *next; +}; + +static BundleAssembly *bundle_assembly_start; +static int bundle_assembly_count; + +void +mono_add_bundled_assembly (const char* name, const unsigned char* data, unsigned int size) +{ + BundleAssembly *entry = g_new0 (BundleAssembly, 1); + entry->assembly.name = g_strdup (name); + g_assert (entry->assembly.name); + entry->assembly.data = data; + entry->assembly.size = size; + entry->next = bundle_assembly_start; + bundle_assembly_start = entry; + ++bundle_assembly_count; +} + +void +mono_register_bundle (void) +{ + MonoBundledAssembly **bundle = g_new0 (MonoBundledAssembly*, bundle_assembly_count + 1); + BundleAssembly *bundle_assembly = bundle_assembly_start; + for (int i = 0; i < bundle_assembly_count; ++i) { + g_assert (bundle_assembly); + bundle [i] = &bundle_assembly->assembly; + bundle_assembly = bundle_assembly->next; + } + mono_register_bundled_assemblies ((const MonoBundledAssembly **)bundle); +} + void mono_aot_init (void) { diff --git a/src/mono/mono/mini/aot-runtime.h b/src/mono/mono/mini/aot-runtime.h index bd666593536276..03b1f86d78b854 100644 --- a/src/mono/mono/mini/aot-runtime.h +++ b/src/mono/mono/mini/aot-runtime.h @@ -248,6 +248,8 @@ typedef struct MonoAotFileInfo /* Number of symbols in the MonoAotFileInfo structure */ #define MONO_AOT_FILE_INFO_NUM_SYMBOLS (((G_STRUCT_OFFSET (MonoAotFileInfo, MONO_AOT_FILE_INFO_LAST_SYMBOL) - G_STRUCT_OFFSET (MonoAotFileInfo, MONO_AOT_FILE_INFO_FIRST_SYMBOL)) / sizeof (gpointer)) + 1) +void mono_add_bundled_assembly (const char* name, const unsigned char* data, unsigned int size); +void mono_register_bundle (void); void mono_aot_init (void); gpointer mono_aot_get_method (MonoMethod *method, MonoError *error); gpointer mono_aot_get_method_from_token (MonoImage *image, guint32 token, MonoError *error); diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index f1803684b2b386..e14c2aa93ef2bb 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -1,4 +1,6 @@ + @@ -17,9 +19,36 @@ <_ExtraLinkerArgs Include="@(_CommonLinkerArgs)" /> + + <_AssembliesToBundleSourceFile>$(BundleDir)bundled_assemblies.c + + + + + + + + <_AssembliesToBundleInternalWithHashes Update="@(_AssembliesToBundleInternalWithHashes)"> + $(BundleDir)%(_AssembliesToBundleInternalWithHashes.Filename)%(_AssembliesToBundleInternalWithHashes.Extension).$([System.String]::Copy(%(_AssembliesToBundleInternalWithHashes.FileHash)).Substring(0, 8)).c + + + + + + + <_ExtraLibrarySources Include="$(_AssembliesToBundleSourceFile)" /> + <_ExtraLibrarySources Include="%(_AssembliesToBundleInternalWithHashes.DestinationFile)" /> + + public string? AssembliesLocation { get; set; } + /// + /// Determines whether or not assemblies are bundled into the library + /// + public bool BundlesAssemblies { get; set; } + public bool StripDebugSymbols { get; set; } /// @@ -309,6 +314,11 @@ private string GenerateExtraDefinitions() extraDefinitions.AppendLine("add_definitions(-DUSES_AOT_DATA=1)"); } + if (BundlesAssemblies) + { + extraDefinitions.AppendLine("add_definitions(-DBUNDLED_ASSEMBLIES=1)"); + } + return extraDefinitions.ToString(); } diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index 6fd37510d00a1c..dc7b45a3f4e900 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -105,7 +105,7 @@ free_aot_data (MonoAssembly *assembly, int size, void *user_data, void *handle) { munmap (handle, size); } -#endif +#endif // USES_AOT_DATA static void runtime_init_callback () @@ -125,6 +125,11 @@ runtime_init_callback () register_aot_modules (); +#if defined(BUNDLED_ASSEMBLIES) + mono_register_assemblies_bundle (); + mono_register_bundle (); +#endif + mono_set_assemblies_path (bundle_path); mono_jit_set_aot_only (true); diff --git a/src/tasks/LibraryBuilder/Templates/library-builder.h b/src/tasks/LibraryBuilder/Templates/library-builder.h index a3fdb705caefee..66c5d4b137bd23 100644 --- a/src/tasks/LibraryBuilder/Templates/library-builder.h +++ b/src/tasks/LibraryBuilder/Templates/library-builder.h @@ -33,9 +33,13 @@ #error Unsupported Host Platform. Ensure the hosting platform is supported by the LibraryBuilder and the appropriate logging functions are added. -#endif +#endif // HOST_ANDROID ^ HOST_APPLE_MOBILE void register_aot_modules (void); +#if defined(BUNDLED_ASSEMBLIES) +void mono_register_assemblies_bundle (void); +void mono_register_bundle (void); +#endif // BUNDLED_ASSEMBLIES void preload_assemblies_with_exported_symbols (); typedef void (*MonoRuntimeInitCallback) (void); void mono_set_runtime_init_callback (MonoRuntimeInitCallback callback); From a34c284727162004102c7cddbba2a32a60ae7bcc Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Mon, 3 Apr 2023 14:02:29 -0400 Subject: [PATCH 006/110] Refactor LibraryBuilder targets to isolate Bundling --- .../msbuild/common/LibraryBuilder.targets | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index e14c2aa93ef2bb..4cba88347ed482 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -5,7 +5,8 @@ AssemblyFile="$(LibraryBuilderTasksAssemblyPath)" /> + Condition="'$(_IsLibraryMode)' == 'true' and '$(RunAOTCompilation)' == 'true'" + DependsOnTargets="_GenerateBundle"> <_IsSharedLibrary>false @@ -16,33 +17,9 @@ <_ExtraLibrarySources Include="$(_AotModuleTablePath)" /> - <_ExtraLinkerArgs Include="@(_CommonLinkerArgs)" /> - - - - <_AssembliesToBundleSourceFile>$(BundleDir)bundled_assemblies.c - - - - - - - - <_AssembliesToBundleInternalWithHashes Update="@(_AssembliesToBundleInternalWithHashes)"> - $(BundleDir)%(_AssembliesToBundleInternalWithHashes.Filename)%(_AssembliesToBundleInternalWithHashes.Extension).$([System.String]::Copy(%(_AssembliesToBundleInternalWithHashes.FileHash)).Substring(0, 8)).c - - - - - - <_ExtraLibrarySources Include="$(_AssembliesToBundleSourceFile)" /> - <_ExtraLibrarySources Include="%(_AssembliesToBundleInternalWithHashes.DestinationFile)" /> + <_ExtraLibrarySources Include="%(_AssembliesToBundleWithHashes.DestinationFile)" /> + <_ExtraLinkerArgs Include="@(_CommonLinkerArgs)" /> + + + + <_AssembliesToBundleSourceFile>$(BundleDir)bundled_assemblies.c + + + + <_AssembliesToBundle Remove="@(_AssembliesToBundle)" /> + <_AssembliesToBundle Include="@(_AssembliesToBundleInternal)" /> + + + + + + + + <_AssembliesToBundleWithHashes Update="@(_AssembliesToBundleWithHashes)"> + $(BundleDir)%(_AssembliesToBundleWithHashes.Filename)%(_AssembliesToBundleWithHashes.Extension).$([System.String]::Copy(%(_AssembliesToBundleWithHashes.FileHash)).Substring(0, 8)).c + + + + + + \ No newline at end of file From c1924b2c7e6c1e02fdf605b636e1e7110157dcba Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Mon, 3 Apr 2023 17:18:12 -0400 Subject: [PATCH 007/110] [mono] Add mono_get_bundled_assembly in preparation for bundled runtimeconfig data --- src/mono/mono/mini/aot-runtime.c | 63 ++++++++++++++++++++------------ src/mono/mono/mini/aot-runtime.h | 3 +- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/src/mono/mono/mini/aot-runtime.c b/src/mono/mono/mini/aot-runtime.c index d0323ea94a1fe3..69fb1648f808a6 100644 --- a/src/mono/mono/mini/aot-runtime.c +++ b/src/mono/mono/mini/aot-runtime.c @@ -2413,39 +2413,56 @@ mono_aot_register_module (gpointer *aot_info) mono_aot_unlock (); } -typedef struct BundleAssembly_ BundleAssembly; +static GHashTable *bundled_assemblies = NULL; -struct BundleAssembly_ { - MonoBundledAssembly assembly; - BundleAssembly *next; -}; +static int bundles_added = 0; -static BundleAssembly *bundle_assembly_start; -static int bundle_assembly_count; +void +mono_add_bundled_assembly (const char *name, const unsigned char *data, unsigned int size) +{ + if (!bundled_assemblies) + bundled_assemblies = g_hash_table_new (g_str_hash, g_str_equal); + + MonoBundledAssembly *bundled_assembly = g_new0 (MonoBundledAssembly, 1); + bundled_assembly->name = g_strdup (name); + g_assert (bundled_assembly->name); + bundled_assembly->data = data; + bundled_assembly->size = size; + g_hash_table_insert (bundled_assemblies, (gpointer) name, bundled_assembly); + bundles_added++; +} void -mono_add_bundled_assembly (const char* name, const unsigned char* data, unsigned int size) +mono_get_bundled_assembly (const char *name, const unsigned char **out_data, unsigned int *out_size) +{ + *out_data = NULL; + *out_size = 0; + MonoBundledAssembly *bundled_assembly; + + if (g_hash_table_lookup_extended (bundled_assemblies, name, NULL, (gpointer *)&bundled_assembly)) { + *out_data = bundled_assembly->data; + *out_size = bundled_assembly->size; + } +} + +static void +populate_bundle (gpointer key, gpointer value, gpointer user_data) { - BundleAssembly *entry = g_new0 (BundleAssembly, 1); - entry->assembly.name = g_strdup (name); - g_assert (entry->assembly.name); - entry->assembly.data = data; - entry->assembly.size = size; - entry->next = bundle_assembly_start; - bundle_assembly_start = entry; - ++bundle_assembly_count; + MonoBundledAssembly *bundled_assembly = (MonoBundledAssembly *)value; + MonoBundledAssembly **bundle = (MonoBundledAssembly **)user_data; + g_assert (bundles_added > 0); + bundle [bundles_added-- - 1] = bundled_assembly; } void mono_register_bundle (void) { - MonoBundledAssembly **bundle = g_new0 (MonoBundledAssembly*, bundle_assembly_count + 1); - BundleAssembly *bundle_assembly = bundle_assembly_start; - for (int i = 0; i < bundle_assembly_count; ++i) { - g_assert (bundle_assembly); - bundle [i] = &bundle_assembly->assembly; - bundle_assembly = bundle_assembly->next; - } + if (!bundled_assemblies) + return; + + MonoBundledAssembly **bundle = g_new0 (MonoBundledAssembly*, bundles_added + 1); + g_hash_table_foreach (bundled_assemblies, populate_bundle, bundle); + mono_register_bundled_assemblies ((const MonoBundledAssembly **)bundle); } diff --git a/src/mono/mono/mini/aot-runtime.h b/src/mono/mono/mini/aot-runtime.h index 03b1f86d78b854..5d980e149e407d 100644 --- a/src/mono/mono/mini/aot-runtime.h +++ b/src/mono/mono/mini/aot-runtime.h @@ -248,7 +248,8 @@ typedef struct MonoAotFileInfo /* Number of symbols in the MonoAotFileInfo structure */ #define MONO_AOT_FILE_INFO_NUM_SYMBOLS (((G_STRUCT_OFFSET (MonoAotFileInfo, MONO_AOT_FILE_INFO_LAST_SYMBOL) - G_STRUCT_OFFSET (MonoAotFileInfo, MONO_AOT_FILE_INFO_FIRST_SYMBOL)) / sizeof (gpointer)) + 1) -void mono_add_bundled_assembly (const char* name, const unsigned char* data, unsigned int size); +void mono_add_bundled_assembly (const char *name, const unsigned char *data, unsigned int size); +void mono_get_bundled_assembly (const char *name, const unsigned char **out_data, unsigned int *out_size); void mono_register_bundle (void); void mono_aot_init (void); gpointer mono_aot_get_method (MonoMethod *method, MonoError *error); From 90842349b3549d495aaf99aa96d34b8cb19cb308 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Mon, 3 Apr 2023 14:03:00 -0400 Subject: [PATCH 008/110] [mono][librarymode] Bundle runtimeconfig.bin --- .../android/build/AndroidBuild.targets | 3 +- .../msbuild/apple/build/AppleBuild.targets | 3 +- .../msbuild/common/LibraryBuilder.targets | 1 + src/tasks/LibraryBuilder/Templates/autoinit.c | 43 ++++++++++++++++--- .../Templates/library-builder.h | 1 + 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/mono/msbuild/android/build/AndroidBuild.targets b/src/mono/msbuild/android/build/AndroidBuild.targets index 0db739451372c2..26cc272768a37e 100644 --- a/src/mono/msbuild/android/build/AndroidBuild.targets +++ b/src/mono/msbuild/android/build/AndroidBuild.targets @@ -247,7 +247,8 @@ + Condition="Exists('$(_AndroidRuntimeConfigFilePath)')" + BeforeTargets="_GenerateBundle"> <_RuntimeConfigReservedProperties Include="RUNTIME_IDENTIFIER"/> <_RuntimeConfigReservedProperties Include="APP_CONTEXT_BASE_DIRECTORY"/> diff --git a/src/mono/msbuild/apple/build/AppleBuild.targets b/src/mono/msbuild/apple/build/AppleBuild.targets index 61b656c3590d1e..9d353fd4e4f938 100644 --- a/src/mono/msbuild/apple/build/AppleBuild.targets +++ b/src/mono/msbuild/apple/build/AppleBuild.targets @@ -274,7 +274,8 @@ + Condition="Exists('$(_AppleRuntimeConfigFilePath)')" + BeforeTargets="_GenerateBundle"> <_RuntimeConfigReservedProperties Include="RUNTIME_IDENTIFIER"/> <_RuntimeConfigReservedProperties Include="APP_CONTEXT_BASE_DIRECTORY"/> diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index 4cba88347ed482..edd689f5763292 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -51,6 +51,7 @@ <_AssembliesToBundle Remove="@(_AssembliesToBundle)" /> <_AssembliesToBundle Include="@(_AssembliesToBundleInternal)" /> + <_AssembliesToBundle Include="$(_ParsedRuntimeConfigFilePath)" /> diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index dc7b45a3f4e900..b32d838528461f 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -18,6 +18,33 @@ #include "library-builder.h" +#if defined(BUNDLED_ASSEMBLIES) + +static void +cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) +{ + free (args); + free (user_data); +} + +static void +initialize_runtimeconfig () +{ + MonovmRuntimeConfigArguments *arg = (MonovmRuntimeConfigArguments *)malloc (sizeof (MonovmRuntimeConfigArguments)); + if (!arg) + LOG_ERROR ("Out of memory.\n"); + + arg->kind = 1; + const char *data; + int data_len; + mono_get_bundled_assembly ("runtimeconfig.bin", &data, &data_len); + arg->runtimeconfig.data.data = data; + arg->runtimeconfig.data.data_len = data_len; + monovm_runtimeconfig_initialize (arg, cleanup_runtime_config, NULL); +} + +#else + static void cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) { @@ -54,6 +81,8 @@ initialize_runtimeconfig (const char *bundle_path) } } +#endif // BUNDLED_ASSEMBLIES + static void initialize_appctx_env_variables (const char *bundle_path) { @@ -110,6 +139,13 @@ free_aot_data (MonoAssembly *assembly, int size, void *user_data, void *handle) static void runtime_init_callback () { + register_aot_modules (); + +#if defined(BUNDLED_ASSEMBLIES) + mono_register_assemblies_bundle (); + mono_register_bundle (); +#endif + const char *assemblies_location = getenv ("%ASSEMBLIES_LOCATION%"); if (!assemblies_location || assemblies_location[0] == '\0') assemblies_location = "./"; @@ -123,13 +159,6 @@ runtime_init_callback () initialize_appctx_env_variables (bundle_path); - register_aot_modules (); - -#if defined(BUNDLED_ASSEMBLIES) - mono_register_assemblies_bundle (); - mono_register_bundle (); -#endif - mono_set_assemblies_path (bundle_path); mono_jit_set_aot_only (true); diff --git a/src/tasks/LibraryBuilder/Templates/library-builder.h b/src/tasks/LibraryBuilder/Templates/library-builder.h index 66c5d4b137bd23..3e1f1cd5f5099d 100644 --- a/src/tasks/LibraryBuilder/Templates/library-builder.h +++ b/src/tasks/LibraryBuilder/Templates/library-builder.h @@ -37,6 +37,7 @@ void register_aot_modules (void); #if defined(BUNDLED_ASSEMBLIES) +void mono_get_bundled_assembly (const char *name, const unsigned char **out_data, unsigned int *out_size); void mono_register_assemblies_bundle (void); void mono_register_bundle (void); #endif // BUNDLED_ASSEMBLIES From 382a93dd00032121df63dc1e283374aec055e183 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Wed, 5 Apr 2023 01:17:28 -0400 Subject: [PATCH 009/110] Address feedback Redesign bundling apis to handle various types of resources Move bundle registering into generated function and handle all forms of bundle registering there Move bundling api into assembly.c and prototypes into assembly-internals.h Rename assemblies to resources in bundling context Include EmitBundle task in wasm nuget Declare macro for case invariant string comparison Add documentation for bundling apis --- src/mono/mono/metadata/assembly-internals.h | 24 +++ src/mono/mono/metadata/assembly.c | 174 ++++++++++++++++++ src/mono/mono/mini/aot-runtime.c | 53 ------ src/mono/mono/mini/aot-runtime.h | 3 - .../msbuild/common/LibraryBuilder.targets | 34 ++-- .../Sdk/Sdk.targets.in | 1 + .../Sdk/Sdk.targets.in | 1 + src/tasks/EmitBundleTask/EmitBundleBase.cs | 30 ++- src/tasks/LibraryBuilder/LibraryBuilder.cs | 6 +- src/tasks/LibraryBuilder/Templates/autoinit.c | 12 +- .../Templates/library-builder.h | 15 +- .../Templates/preloaded-assemblies.c | 2 +- 12 files changed, 264 insertions(+), 91 deletions(-) diff --git a/src/mono/mono/metadata/assembly-internals.h b/src/mono/mono/metadata/assembly-internals.h index b5e18fa99dfc15..2f94bf3d2d9e62 100644 --- a/src/mono/mono/metadata/assembly-internals.h +++ b/src/mono/mono/metadata/assembly-internals.h @@ -141,6 +141,30 @@ mono_assembly_get_image_internal (MonoAssembly *assembly); void mono_set_assemblies_path_direct (char **path); +typedef enum { + BUNDLED_BLOB, + BUNDLED_ASSEMBLY, + BUNDLED_SATELLITE_ASSEMBLY, + BUNDLED_BINARY, + BUNDLED_RESOURCE_COUNT, +} BundledResourceType; + +typedef struct BundledResource { + const char *culture; // Satellite assemblies + const unsigned char *data; + unsigned int size; + BundledResourceType type; +} BundledResource; + +void +mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, BundledResourceType type); + +void +mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size); + +void +mono_register_bundled_resources (void); + MONO_COMPONENT_API gboolean mono_assembly_is_jit_optimizer_disabled (MonoAssembly *assembly); diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 6375754f97f04e..7dc34384925236 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -3156,6 +3156,20 @@ mono_assembly_get_name_internal (MonoAssembly *assembly) return &assembly->aname; } +/** + * mono_create_new_bundled_assembly: + */ +static MonoBundledAssembly * +mono_create_new_bundled_assembly (const char *name, const unsigned char *data, unsigned int size) +{ + MonoBundledAssembly *assembly = g_new0 (MonoBundledAssembly, 1); + assembly->name = strdup (name); + g_assert (assembly->name); + assembly->data = data; + assembly->size = size; + return assembly; +} + /** * mono_register_bundled_assemblies: */ @@ -3173,7 +3187,9 @@ mono_create_new_bundled_satellite_assembly (const char *name, const char *cultur { MonoBundledSatelliteAssembly *satellite_assembly = g_new0 (MonoBundledSatelliteAssembly, 1); satellite_assembly->name = strdup (name); + g_assert (satellite_assembly->name); satellite_assembly->culture = strdup (culture); + g_assert (satellite_assembly->culture); satellite_assembly->data = data; satellite_assembly->size = size; return satellite_assembly; @@ -3188,6 +3204,164 @@ mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly * satellite_bundles = assemblies; } +static GHashTable *bundled_resources = NULL; + +static int *resources_bundled_counts = NULL; + +//--------------------------------------------------------------------------------------- +// +// mono_add_bundled_resource handles bundling of many types of resources to circumvent +// needing to find or have those resources on disk. The BundledResource struct models +// the union of information carried by all supported types of resources which are +// enumerated in BundledResourceType. +// +// bundled_resources: +// A single hash table will hold all resources being bundled with the understanding +// that all resources being bundled are uniquely named. The bundled resource is tagged +// with the type of resource it represents for later processing of specific types of +// resources as done in mono_register_bundled_resources. +// +// resources_bundled_counts: +// An integer array tracks the number of each resource type added to the bundle +// to facilitate appropriate memory allocation when processing the bundle as done in +// mono_register_bundled_resources +// +// Arguments: +// * name - Unique name of the resource +// * culture - Culture of satellite assemblies +// * data - Byte data array of the bundled resource +// * size - Size of byte data array of the bundled resource +// type - Type of bundled resource +// + +void +mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, BundledResourceType type) +{ + if (!bundled_resources) + bundled_resources = g_hash_table_new (g_str_hash, g_str_equal); + + if (!resources_bundled_counts) + resources_bundled_counts = g_malloc0 (BUNDLED_RESOURCE_COUNT * sizeof (int)); + + g_assert (!g_hash_table_contains (bundled_resources, name)); + + BundledResource *bundled_resource = g_new0 (BundledResource, 1); + bundled_resource->culture = culture; + bundled_resource->data = data; + bundled_resource->size = size; + bundled_resource->type = type; + ++resources_bundled_counts[type]; + g_hash_table_insert (bundled_resources, (gpointer) name, bundled_resource); +} + +//--------------------------------------------------------------------------------------- +// +// mono_get_bundled_resource_data retrieves the data of the bundled resource. If the +// requested bundled resource's name has been added via mono_add_bundled_resource, +// the byte array data and data size are populated. +// +// Arguments: +// * name - Unique name of the resource +// ** out_data - the pointer to the byte array data +// ** out_size - the pointer to the data size + +void +mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size) +{ + *out_data = NULL; + *out_size = 0; + + BundledResource *bundled_resource; + + if (g_hash_table_lookup_extended (bundled_resources, name, NULL, (gpointer *)&bundled_resource)) { + *out_data = bundled_resource->data; + *out_size = bundled_resource->size; + } +} + +static int num_bundled_resources_to_register = 0; + +//--------------------------------------------------------------------------------------- +// +// populate_bundled_assemblies assists in populating the MonoBundledAssembly +// to register in mono_register_bundled_resources +// + +static void +populate_bundled_assemblies (gpointer key, gpointer value, gpointer user_data) +{ + MonoBundledAssembly **bundle = (MonoBundledAssembly **)user_data; + BundledResource *bundled_resource = (BundledResource *)value; + if (!bundle || !bundled_resource || bundled_resource->type != BUNDLED_ASSEMBLY) + return; + + MonoBundledAssembly *bundled_assembly = mono_create_new_bundled_assembly ((const char *)key, bundled_resource->data, bundled_resource->size); + + g_assert (num_bundled_resources_to_register < resources_bundled_counts[BUNDLED_ASSEMBLY]); + bundle [num_bundled_resources_to_register++] = bundled_assembly; +} + +//--------------------------------------------------------------------------------------- +// +// populate_bundled_satellite_assemblies assists in populating the MonoBundledSatelliteAssembly +// to register in mono_register_bundled_resources +// + +static void +populate_bundled_satellite_assemblies (gpointer key, gpointer value, gpointer user_data) +{ + MonoBundledSatelliteAssembly **bundle = (MonoBundledSatelliteAssembly **)user_data; + BundledResource *bundled_resource = (BundledResource *)value; + if (!bundle || !bundled_resource || bundled_resource->type != BUNDLED_SATELLITE_ASSEMBLY) + return; + + MonoBundledSatelliteAssembly *bundled_satellite_assembly = mono_create_new_bundled_satellite_assembly ((const char *)key, bundled_resource->culture, bundled_resource->data, bundled_resource->size); + + g_assert (num_bundled_resources_to_register < resources_bundled_counts[BUNDLED_SATELLITE_ASSEMBLY]); + bundle [num_bundled_resources_to_register++] = bundled_satellite_assembly; +} + +//--------------------------------------------------------------------------------------- +// +// mono_register_bundled_resources registers resources added by mono_add_bundled_resource. +// Out of the supported resource types in BundledResourceType, assemblies and satellite +// assemblies are registered. +// +// To appropriately allocate memory for the MonoBundledAssembly array and +// MonoBundledSatelliteAssembly array to register, resources_bundled_counts is leveraged, +// and therefore, mono_add_bundled_resource must be previously invoked for any successful +// registering of assemblies and satellite assemblies. +// + +void +mono_register_bundled_resources (void) +{ + if (!bundled_resources) + return; + + // register assemblies + if (resources_bundled_counts [BUNDLED_ASSEMBLY] != 0) + { + MonoBundledAssembly **assembly_bundle = g_new0 (MonoBundledAssembly*, resources_bundled_counts[BUNDLED_ASSEMBLY] + 1); + num_bundled_resources_to_register = 0; + g_hash_table_foreach (bundled_resources, populate_bundled_assemblies, assembly_bundle); + g_assert (num_bundled_resources_to_register == resources_bundled_counts[BUNDLED_ASSEMBLY]); + + mono_register_bundled_assemblies ((const MonoBundledAssembly **)assembly_bundle); + } + + // register satellite assemblies + if (resources_bundled_counts [BUNDLED_SATELLITE_ASSEMBLY] != 0) + { + MonoBundledSatelliteAssembly **satellite_assembly_bundle = g_new0 (MonoBundledSatelliteAssembly *, resources_bundled_counts[BUNDLED_SATELLITE_ASSEMBLY] + 1); + num_bundled_resources_to_register = 0; + g_hash_table_foreach (bundled_resources, populate_bundled_satellite_assemblies, satellite_assembly_bundle); + g_assert (num_bundled_resources_to_register == resources_bundled_counts[BUNDLED_SATELLITE_ASSEMBLY]); + + mono_register_bundled_satellite_assemblies ((const MonoBundledSatelliteAssembly **)satellite_assembly_bundle); + } +} + /** * mono_assembly_is_jit_optimizer_disabled: * diff --git a/src/mono/mono/mini/aot-runtime.c b/src/mono/mono/mini/aot-runtime.c index 69fb1648f808a6..46e0aff893914f 100644 --- a/src/mono/mono/mini/aot-runtime.c +++ b/src/mono/mono/mini/aot-runtime.c @@ -2413,59 +2413,6 @@ mono_aot_register_module (gpointer *aot_info) mono_aot_unlock (); } -static GHashTable *bundled_assemblies = NULL; - -static int bundles_added = 0; - -void -mono_add_bundled_assembly (const char *name, const unsigned char *data, unsigned int size) -{ - if (!bundled_assemblies) - bundled_assemblies = g_hash_table_new (g_str_hash, g_str_equal); - - MonoBundledAssembly *bundled_assembly = g_new0 (MonoBundledAssembly, 1); - bundled_assembly->name = g_strdup (name); - g_assert (bundled_assembly->name); - bundled_assembly->data = data; - bundled_assembly->size = size; - g_hash_table_insert (bundled_assemblies, (gpointer) name, bundled_assembly); - bundles_added++; -} - -void -mono_get_bundled_assembly (const char *name, const unsigned char **out_data, unsigned int *out_size) -{ - *out_data = NULL; - *out_size = 0; - MonoBundledAssembly *bundled_assembly; - - if (g_hash_table_lookup_extended (bundled_assemblies, name, NULL, (gpointer *)&bundled_assembly)) { - *out_data = bundled_assembly->data; - *out_size = bundled_assembly->size; - } -} - -static void -populate_bundle (gpointer key, gpointer value, gpointer user_data) -{ - MonoBundledAssembly *bundled_assembly = (MonoBundledAssembly *)value; - MonoBundledAssembly **bundle = (MonoBundledAssembly **)user_data; - g_assert (bundles_added > 0); - bundle [bundles_added-- - 1] = bundled_assembly; -} - -void -mono_register_bundle (void) -{ - if (!bundled_assemblies) - return; - - MonoBundledAssembly **bundle = g_new0 (MonoBundledAssembly*, bundles_added + 1); - g_hash_table_foreach (bundled_assemblies, populate_bundle, bundle); - - mono_register_bundled_assemblies ((const MonoBundledAssembly **)bundle); -} - void mono_aot_init (void) { diff --git a/src/mono/mono/mini/aot-runtime.h b/src/mono/mono/mini/aot-runtime.h index 5d980e149e407d..bd666593536276 100644 --- a/src/mono/mono/mini/aot-runtime.h +++ b/src/mono/mono/mini/aot-runtime.h @@ -248,9 +248,6 @@ typedef struct MonoAotFileInfo /* Number of symbols in the MonoAotFileInfo structure */ #define MONO_AOT_FILE_INFO_NUM_SYMBOLS (((G_STRUCT_OFFSET (MonoAotFileInfo, MONO_AOT_FILE_INFO_LAST_SYMBOL) - G_STRUCT_OFFSET (MonoAotFileInfo, MONO_AOT_FILE_INFO_FIRST_SYMBOL)) / sizeof (gpointer)) + 1) -void mono_add_bundled_assembly (const char *name, const unsigned char *data, unsigned int size); -void mono_get_bundled_assembly (const char *name, const unsigned char **out_data, unsigned int *out_size); -void mono_register_bundle (void); void mono_aot_init (void); gpointer mono_aot_get_method (MonoMethod *method, MonoError *error); gpointer mono_aot_get_method_from_token (MonoImage *image, guint32 token, MonoError *error); diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index edd689f5763292..59f884443fd82f 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -17,15 +17,15 @@ <_ExtraLibrarySources Include="$(_AotModuleTablePath)" /> - <_ExtraLibrarySources Include="$(_AssembliesToBundleSourceFile)" /> - <_ExtraLibrarySources Include="%(_AssembliesToBundleWithHashes.DestinationFile)" /> + <_ExtraLibrarySources Include="$(_ResourcesToBundleSourceFile)" /> + <_ExtraLibrarySources Include="%(_ResourcesToBundleWithHashes.DestinationFile)" /> <_ExtraLinkerArgs Include="@(_CommonLinkerArgs)" /> + Condition="'$(BundlesResources)' == 'true'"> - <_AssembliesToBundleSourceFile>$(BundleDir)bundled_assemblies.c + <_ResourcesToBundleSourceFile>$(BundleDir)bundled_resources.c - <_AssembliesToBundle Remove="@(_AssembliesToBundle)" /> - <_AssembliesToBundle Include="@(_AssembliesToBundleInternal)" /> - <_AssembliesToBundle Include="$(_ParsedRuntimeConfigFilePath)" /> + <_ResourcesToBundle Remove="@(_ResourcesToBundle)" /> + <_ResourcesToBundle Include="@(_AssembliesToBundleInternal)" /> + <_ResourcesToBundle Include="$(_ParsedRuntimeConfigFilePath)" /> - - + + - <_AssembliesToBundleWithHashes Update="@(_AssembliesToBundleWithHashes)"> - $(BundleDir)%(_AssembliesToBundleWithHashes.Filename)%(_AssembliesToBundleWithHashes.Extension).$([System.String]::Copy(%(_AssembliesToBundleWithHashes.FileHash)).Substring(0, 8)).c - + <_ResourcesToBundleWithHashes Update="@(_ResourcesToBundleWithHashes)"> + $(BundleDir)%(_ResourcesToBundleWithHashes.Filename)%(_ResourcesToBundleWithHashes.Extension).$([System.String]::Copy(%(_ResourcesToBundleWithHashes.FileHash)).Substring(0, 8)).c + diff --git a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/Sdk.targets.in b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/Sdk.targets.in index 0f9edfb14f10d3..f4bbdd7376df65 100644 --- a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/Sdk.targets.in +++ b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/Sdk.targets.in @@ -4,6 +4,7 @@ <_TasksDir Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETCoreTasks}\ <_TasksDir Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETFrameworkTasks}\ + $(_TasksDir)EmitBundle.dll $(_TasksDir)WasmAppBuilder.dll $(_TasksDir)WasmBuildTasks.dll $([MSBuild]::NormalizeDirectory($(MSBuildThisFileDirectory), '..', 'WasmAppHost')) diff --git a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/Sdk/Sdk.targets.in b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/Sdk/Sdk.targets.in index 9fd83ffe152b55..e53a67141e6da0 100644 --- a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/Sdk/Sdk.targets.in +++ b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/Sdk/Sdk.targets.in @@ -4,6 +4,7 @@ <_TasksDir Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETCoreTasks}\ <_TasksDir Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETFrameworkTasks}\ + $(_TasksDir)EmitBundle.dll $(_TasksDir)WasmAppBuilder.dll $(_TasksDir)WasmBuildTasks.dll $([MSBuild]::NormalizeDirectory($(MSBuildThisFileDirectory), '..', 'WasmAppHost')) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index dfd84a74b1a23c..3e1a0d32e9c6f9 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -60,7 +60,8 @@ public override bool Execute() var outputFile = registeredFile.GetMetadata("DestinationFile"); var registeredName = group.Key; var symbolName = ToSafeSymbolName(outputFile); - return (registeredName, symbolName); + var fileType = GetFileType(registeredFile.ItemSpec); + return (registeredName, symbolName, fileType); }).ToList(); Log.LogMessage(MessageImportance.Low, "Bundling {0} files for {1}", files.Count, BundleName); @@ -148,9 +149,11 @@ private static byte[] InitLookupTable() public abstract bool Emit(string destinationFile, Action inputProvider); - public static void GenerateRegisteredBundledObjects(string newFunctionName, string callbackFunctionName, ICollection<(string registeredName, string symbol)> files, StreamWriter outputUtf8Writer) + public static void GenerateRegisteredBundledObjects(string newFunctionName, string callbackFunctionName, ICollection<(string registeredName, string symbol, string type)> files, StreamWriter outputUtf8Writer) { - outputUtf8Writer.WriteLine($"int {callbackFunctionName}(const char* name, const unsigned char* data, unsigned int size);"); + outputUtf8Writer.WriteLine("typedef enum {\n BUNDLED_BLOB,\n BUNDLED_ASSEMBLY,\n BUNDLED_SATELLITE_ASSEMBLY,\n BUNDLED_BINARY,\n BUNDLED_RESOURCE_COUNT,\n} BundledResourceType;"); + outputUtf8Writer.WriteLine($"int {callbackFunctionName}(const char *name, const char *culture, const unsigned char *data, unsigned int size, BundledResourceType type);"); + outputUtf8Writer.WriteLine($"void mono_register_bundled_resources (void);"); outputUtf8Writer.WriteLine(); foreach (var tuple in files) @@ -164,9 +167,11 @@ public static void GenerateRegisteredBundledObjects(string newFunctionName, stri foreach (var tuple in files) { - outputUtf8Writer.WriteLine($" {callbackFunctionName} (\"{tuple.registeredName}\", {tuple.symbol}, {tuple.symbol}_len);"); + outputUtf8Writer.WriteLine($" {callbackFunctionName} (\"{tuple.registeredName}\", 0, {tuple.symbol}, {tuple.symbol}_len, {tuple.type});"); } + outputUtf8Writer.WriteLine($" mono_register_bundled_resources ();"); + outputUtf8Writer.WriteLine("}"); } @@ -227,6 +232,23 @@ private static string ToSafeSymbolName(string destinationFileName) return sb.ToString(); } + private static string GetFileType(string destinationFileName) + { + if (destinationFileName.EndsWith(".resources.dll")) + { + return "BUNDLED_SATELLITE_ASSEMBLY"; + } + if (destinationFileName.EndsWith(".dll")) + { + return "BUNDLED_ASSEMBLY"; + } + if (destinationFileName.EndsWith(".bin")) + { + return "BUNDLED_BINARY"; + } + return "BUNDLED_BLOB"; + } + // Equivalent to "isalnum" private static bool IsAlphanumeric(char c) => c is (>= 'a' and <= 'z') diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index 78239b84f9e392..da4010efc06c8c 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -88,7 +88,7 @@ public bool IsSharedLibrary /// /// Determines whether or not assemblies are bundled into the library /// - public bool BundlesAssemblies { get; set; } + public bool BundlesResources { get; set; } public bool StripDebugSymbols { get; set; } @@ -314,9 +314,9 @@ private string GenerateExtraDefinitions() extraDefinitions.AppendLine("add_definitions(-DUSES_AOT_DATA=1)"); } - if (BundlesAssemblies) + if (BundlesResources) { - extraDefinitions.AppendLine("add_definitions(-DBUNDLED_ASSEMBLIES=1)"); + extraDefinitions.AppendLine("add_definitions(-DBUNDLED_RESOURCES=1)"); } return extraDefinitions.ToString(); diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index b32d838528461f..d33f9689c57b6e 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -18,7 +18,7 @@ #include "library-builder.h" -#if defined(BUNDLED_ASSEMBLIES) +#if defined(BUNDLED_RESOURCES) static void cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) @@ -37,7 +37,8 @@ initialize_runtimeconfig () arg->kind = 1; const char *data; int data_len; - mono_get_bundled_assembly ("runtimeconfig.bin", &data, &data_len); + mono_get_bundled_resource_data ("runtimeconfig.bin", &data, &data_len); + arg->runtimeconfig.name = NULL; arg->runtimeconfig.data.data = data; arg->runtimeconfig.data.data_len = data_len; monovm_runtimeconfig_initialize (arg, cleanup_runtime_config, NULL); @@ -81,7 +82,7 @@ initialize_runtimeconfig (const char *bundle_path) } } -#endif // BUNDLED_ASSEMBLIES +#endif // BUNDLED_RESOURCES static void initialize_appctx_env_variables (const char *bundle_path) @@ -141,9 +142,8 @@ runtime_init_callback () { register_aot_modules (); -#if defined(BUNDLED_ASSEMBLIES) - mono_register_assemblies_bundle (); - mono_register_bundle (); +#if defined(BUNDLED_RESOURCES) + mono_register_resources_bundle (); #endif const char *assemblies_location = getenv ("%ASSEMBLIES_LOCATION%"); diff --git a/src/tasks/LibraryBuilder/Templates/library-builder.h b/src/tasks/LibraryBuilder/Templates/library-builder.h index 3e1f1cd5f5099d..0f3065f497e26b 100644 --- a/src/tasks/LibraryBuilder/Templates/library-builder.h +++ b/src/tasks/LibraryBuilder/Templates/library-builder.h @@ -7,6 +7,7 @@ #include #include +// Logging #if defined(HOST_ANDROID) #include @@ -33,13 +34,19 @@ #error Unsupported Host Platform. Ensure the hosting platform is supported by the LibraryBuilder and the appropriate logging functions are added. -#endif // HOST_ANDROID ^ HOST_APPLE_MOBILE +#endif // Logging + +// Platform specific native functions +#if defined(HOST_WINDOWS) +#define STR_CASE_CMP _stricmp +#else +#define STR_CASE_CMP strcasecmp +#endif // Platform specific native functions void register_aot_modules (void); #if defined(BUNDLED_ASSEMBLIES) -void mono_get_bundled_assembly (const char *name, const unsigned char **out_data, unsigned int *out_size); -void mono_register_assemblies_bundle (void); -void mono_register_bundle (void); +void mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size); +void mono_register_resources_bundle (void); #endif // BUNDLED_ASSEMBLIES void preload_assemblies_with_exported_symbols (); typedef void (*MonoRuntimeInitCallback) (void); diff --git a/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c b/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c index 4f949ff788b6c7..00e6a99c7a9620 100644 --- a/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c +++ b/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c @@ -18,7 +18,7 @@ preload_assembly (const char* filename) if (!filename_without_extension) LOG_ERROR ("Out of memory.\n"); - if (!strcasecmp (".dll", &filename [len - 4])) + if (!STR_CASE_CMP (".dll", &filename [len - 4])) *(filename_without_extension + len - 4) = '\0'; assembly = mono_assembly_load_with_partial_name (filename_without_extension, NULL); From d4d4193286f732a9f1eab3c6ada53742c7e85a9b Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 6 Apr 2023 16:55:42 -0400 Subject: [PATCH 010/110] Consolidate runtimeconfig functions --- src/tasks/LibraryBuilder/Templates/autoinit.c | 67 +++++++------------ 1 file changed, 25 insertions(+), 42 deletions(-) diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index d33f9689c57b6e..c88abd7fac6f58 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -18,71 +18,54 @@ #include "library-builder.h" -#if defined(BUNDLED_RESOURCES) - static void cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) { + if (args->kind == 0) + free ((void *)args->runtimeconfig.name.path); + free (args); free (user_data); } static void -initialize_runtimeconfig () +initialize_runtimeconfig (const char *bundle_path) { MonovmRuntimeConfigArguments *arg = (MonovmRuntimeConfigArguments *)malloc (sizeof (MonovmRuntimeConfigArguments)); if (!arg) LOG_ERROR ("Out of memory.\n"); - arg->kind = 1; + const char *file_name = "runtimeconfig.bin"; const char *data; int data_len; - mono_get_bundled_resource_data ("runtimeconfig.bin", &data, &data_len); - arg->runtimeconfig.name = NULL; - arg->runtimeconfig.data.data = data; - arg->runtimeconfig.data.data_len = data_len; - monovm_runtimeconfig_initialize (arg, cleanup_runtime_config, NULL); -} - -#else - -static void -cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) -{ - free ((void *)args->runtimeconfig.name.path); - free (args); - free (user_data); -} + mono_get_bundled_resource_data (file_name, &data, &data_len); -static void -initialize_runtimeconfig (const char *bundle_path) -{ - char *file_name = "runtimeconfig.bin"; - size_t str_len = sizeof (char) * (strlen (bundle_path) + strlen (file_name) + 2); // +1 "/", +1 null-terminating char - char *file_path = (char *)malloc (str_len); - if (!file_path) - LOG_ERROR ("Out of memory.\n"); - - int num_char = snprintf (file_path, str_len, "%s/%s", bundle_path, file_name); - if (num_char <= 0 || num_char >= str_len) - LOG_ERROR ("Encoding error while formatting '%s' and '%s' into \"%%s/%%s\".\n", bundle_path, file_name); + if (data) { + arg->kind = 1; + arg->runtimeconfig.data.data = data; + arg->runtimeconfig.data.data_len = data_len; + } else { + size_t str_len = sizeof (char) * (strlen (bundle_path) + strlen (file_name) + 2); // +1 "/", +1 null-terminating char + char *file_path = (char *)malloc (str_len); + if (!file_path) + LOG_ERROR ("Out of memory.\n"); - struct stat buffer; + int num_char = snprintf (file_path, str_len, "%s/%s", bundle_path, file_name); + if (num_char <= 0 || num_char >= str_len) + LOG_ERROR ("Encoding error while formatting '%s' and '%s' into \"%%s/%%s\".\n", bundle_path, file_name); - if (stat (file_path, &buffer) == 0) { - MonovmRuntimeConfigArguments *arg = (MonovmRuntimeConfigArguments *)malloc (sizeof (MonovmRuntimeConfigArguments)); - if (!arg) - LOG_ERROR ("Out of memory.\n"); + struct stat buffer; + if (stat (file_path, &buffer) != 0) { + free (file_path); + return; + } arg->kind = 0; arg->runtimeconfig.name.path = file_path; - monovm_runtimeconfig_initialize (arg, cleanup_runtime_config, NULL); - } else { - free (file_path); } -} -#endif // BUNDLED_RESOURCES + monovm_runtimeconfig_initialize (arg, cleanup_runtime_config, NULL); +} static void initialize_appctx_env_variables (const char *bundle_path) From e70e61a8132a3b39c46c196fb9c61b7a73b485e1 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 6 Apr 2023 17:09:53 -0400 Subject: [PATCH 011/110] [LibraryBuilder] Enable bundling by default --- src/mono/msbuild/common/LibraryBuilder.targets | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index 59f884443fd82f..943148317ffd72 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -13,6 +13,8 @@ <_IsSharedLibrary Condition="'$(NativeLib)' == 'shared'">true <_UsesCustomRuntimeInitCallback>false <_UsesCustomRuntimeInitCallback Condition="$(CustomRuntimeInitCallback) != ''">true + <_BundlesResources>$(BundlesResources) + <_BundlesResources Condition="'$(_BundlesResources)' == ''">true @@ -25,7 +27,7 @@ + Condition="'$(BundlesResources)' != 'false'"> <_ResourcesToBundleSourceFile>$(BundleDir)bundled_resources.c From 791c743f2b9f42176255c2fa6a7e3768bc7738f3 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 11 Apr 2023 16:45:28 -0400 Subject: [PATCH 012/110] Handle pdb bundling --- src/mono/mono/metadata/assembly-internals.h | 1 + src/mono/mono/metadata/assembly.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/mono/mono/metadata/assembly-internals.h b/src/mono/mono/metadata/assembly-internals.h index 2f94bf3d2d9e62..ca3a211a441a02 100644 --- a/src/mono/mono/metadata/assembly-internals.h +++ b/src/mono/mono/metadata/assembly-internals.h @@ -146,6 +146,7 @@ typedef enum { BUNDLED_ASSEMBLY, BUNDLED_SATELLITE_ASSEMBLY, BUNDLED_BINARY, + BUNDLED_PDB, BUNDLED_RESOURCE_COUNT, } BundledResourceType; diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 55b3fc8942d8ca..e31dd2ff81740e 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -3237,6 +3237,12 @@ static int *resources_bundled_counts = NULL; void mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, BundledResourceType type) { + if (type == BUNDLED_PDB) + { + mono_register_symfile_for_assembly (name, data, size); + return; + } + if (!bundled_resources) bundled_resources = g_hash_table_new (g_str_hash, g_str_equal); From d98c0a58ce205c8085f872e97a5b6da5735eb09f Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 11 Apr 2023 18:13:54 -0400 Subject: [PATCH 013/110] Default EmitBundle callback function to mono_add_bundled_resource --- src/mono/mono/utils/mono-dl-wasm.c | 42 ------------------- src/mono/mono/utils/mono-dl.h | 5 --- .../msbuild/common/LibraryBuilder.targets | 1 - src/mono/wasi/build/WasiApp.Native.targets | 1 - src/mono/wasi/wasi.proj | 4 +- src/mono/wasm/wasm.proj | 1 - src/native/libs/System.Native/pal_datetime.c | 6 ++- src/tasks/EmitBundleTask/EmitBundleBase.cs | 11 ++--- 8 files changed, 9 insertions(+), 62 deletions(-) diff --git a/src/mono/mono/utils/mono-dl-wasm.c b/src/mono/mono/utils/mono-dl-wasm.c index 774d1ba6dd2b13..1490b086ccfc82 100644 --- a/src/mono/mono/utils/mono-dl-wasm.c +++ b/src/mono/mono/utils/mono-dl-wasm.c @@ -94,45 +94,3 @@ mono_dl_close_handle (MonoDl *module, MonoError *error) MONO_EMPTY_SOURCE_FILE (mono_dl_wasm); #endif - -#if defined (HOST_WASM) - -static GHashTable *name_to_blob = NULL; - -typedef struct { - const unsigned char *data; - unsigned int size; -} FileBlob; - -int -mono_wasm_add_bundled_file (const char *name, const unsigned char *data, unsigned int size) -{ - // printf("mono_wasm_add_bundled_file: %s %p %d\n", name, data, size); - if(name_to_blob == NULL) - { - name_to_blob = g_hash_table_new (g_str_hash, g_str_equal); - } - FileBlob *blob = g_new0 (FileBlob, 1); - blob->data = data; - blob->size = size; - g_hash_table_insert (name_to_blob, (gpointer) name, blob); - return 0; -} - -const unsigned char* -mono_wasm_get_bundled_file (const char *name, int* out_length) -{ - FileBlob *blob = (FileBlob *)g_hash_table_lookup (name_to_blob, name); - if (blob != NULL) - { - // printf("mono_wasm_get_bundled_file: %s %p %d \n", name, blob->data, blob->size); - *out_length = blob->size; - return blob->data; - } - - // printf("mono_wasm_get_bundled_file: %s not found \n", name); - *out_length = 0; - return NULL; -} - -#endif /* HOST_WASM */ diff --git a/src/mono/mono/utils/mono-dl.h b/src/mono/mono/utils/mono-dl.h index d3893ff8a71390..40d5fff11d094e 100644 --- a/src/mono/mono/utils/mono-dl.h +++ b/src/mono/mono/utils/mono-dl.h @@ -58,10 +58,5 @@ int mono_dl_convert_flags (int mono_flags, int native_flags); char* mono_dl_current_error_string (void); const char* mono_dl_get_system_dir (void); -#if defined (HOST_WASM) -int mono_wasm_add_bundled_file (const char *name, const unsigned char *data, unsigned int size); -const unsigned char* mono_wasm_get_bundled_file (const char *name, int* out_length); -#endif /* HOST_WASM */ - #endif /* __MONO_UTILS_DL_H__ */ diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index f72480fa442ffb..dfb9d482af74bd 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -71,7 +71,6 @@ FilesToBundle="@(_ResourcesToBundleWithHashes)" BundleName="resources" BundleFile="$(_ResourcesToBundleSourceFile)" - RegistrationCallbackFunctionName="mono_add_bundled_resource" /> diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 3a983159f3c34b..ef40bafc18fe26 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -362,7 +362,6 @@ ClangExecutable="$(WasiClang)" BundleName="assemblies" BundleFile="$(_WasmAssembliesBundleObjectFile)" - RegistrationCallbackFunctionName="mono_wasm_add_assembly" /> diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index 2da5234d267b3c..ab57d08283d29f 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -91,9 +91,7 @@ ClangExecutable="$(WasiClang)" BundleName="timezones" BundleFile="$(_WasmTimezonesBundleObjectFile)" - RegistrationCallbackFunctionName="mono_wasm_add_bundled_file" - > - + /> diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index 826e014e77aea9..016107045d4883 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -108,7 +108,6 @@ FilesToBundle="@(_WasmBundleTimezonesWithHashes)" BundleName="timezones" BundleFile="$(WasmObjDir)\wasm-bundled-timezones.c" - RegistrationCallbackFunctionName="mono_wasm_add_bundled_file" /> diff --git a/src/native/libs/System.Native/pal_datetime.c b/src/native/libs/System.Native/pal_datetime.c index 61b0dba23a9195..739e456ea7c709 100644 --- a/src/native/libs/System.Native/pal_datetime.c +++ b/src/native/libs/System.Native/pal_datetime.c @@ -21,7 +21,7 @@ static const int64_t TICKS_PER_MICROSECOND = 10; /* 1000 / 100 */ #endif #if defined(TARGET_WASI) || defined(TARGET_BROWSER) -extern const unsigned char* mono_wasm_get_bundled_file(const char* name, int* out_length); +extern void mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size) #endif // @@ -67,7 +67,9 @@ const char* SystemNative_GetTimeZoneData(const char* name, int* length) assert(name != NULL); assert(length != NULL); #if defined(TARGET_WASI) || defined(TARGET_BROWSER) - return (const char*) mono_wasm_get_bundled_file(name, length); + const char *data; + mono_get_bundled_resource_data (name, &out_data, length) + return data; #else assert_msg(false, "Not supported on this platform", 0); (void)name; // unused diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 3e1a0d32e9c6f9..8ca0ce57a0f774 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -27,9 +27,6 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab [Required] public string BundleFile { get; set; } = default!; - [Required] - public string RegistrationCallbackFunctionName { get; set; } = default!; - public override bool Execute() { // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore @@ -108,7 +105,7 @@ public override bool Execute() return Emit(BundleFile, (inputStream) => { using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); - GenerateRegisteredBundledObjects($"mono_register_{BundleName}_bundle", RegistrationCallbackFunctionName, files, outputUtf8Writer); + GenerateRegisteredBundledObjects($"mono_register_{BundleName}_bundle", files, outputUtf8Writer); }) && !Log.HasLoggedErrors; } @@ -149,10 +146,10 @@ private static byte[] InitLookupTable() public abstract bool Emit(string destinationFile, Action inputProvider); - public static void GenerateRegisteredBundledObjects(string newFunctionName, string callbackFunctionName, ICollection<(string registeredName, string symbol, string type)> files, StreamWriter outputUtf8Writer) + public static void GenerateRegisteredBundledObjects(string newFunctionName, ICollection<(string registeredName, string symbol, string type)> files, StreamWriter outputUtf8Writer) { outputUtf8Writer.WriteLine("typedef enum {\n BUNDLED_BLOB,\n BUNDLED_ASSEMBLY,\n BUNDLED_SATELLITE_ASSEMBLY,\n BUNDLED_BINARY,\n BUNDLED_RESOURCE_COUNT,\n} BundledResourceType;"); - outputUtf8Writer.WriteLine($"int {callbackFunctionName}(const char *name, const char *culture, const unsigned char *data, unsigned int size, BundledResourceType type);"); + outputUtf8Writer.WriteLine($"void mono_add_bundled_resource(const char *name, const char *culture, const unsigned char *data, unsigned int size, BundledResourceType type);"); outputUtf8Writer.WriteLine($"void mono_register_bundled_resources (void);"); outputUtf8Writer.WriteLine(); @@ -167,7 +164,7 @@ public static void GenerateRegisteredBundledObjects(string newFunctionName, stri foreach (var tuple in files) { - outputUtf8Writer.WriteLine($" {callbackFunctionName} (\"{tuple.registeredName}\", 0, {tuple.symbol}, {tuple.symbol}_len, {tuple.type});"); + outputUtf8Writer.WriteLine($" mono_add_bundled_resource (\"{tuple.registeredName}\", 0, {tuple.symbol}, {tuple.symbol}_len, {tuple.type});"); } outputUtf8Writer.WriteLine($" mono_register_bundled_resources ();"); From 6fc54b1e7db2b5b5905ec66dfc95d2ff6ddb9f73 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 11 Apr 2023 22:17:47 -0400 Subject: [PATCH 014/110] Combine bundled resource types into generic bundled data --- src/mono/mono/metadata/assembly-internals.h | 3 +-- src/tasks/EmitBundleTask/EmitBundleBase.cs | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/mono/mono/metadata/assembly-internals.h b/src/mono/mono/metadata/assembly-internals.h index ca3a211a441a02..96be75fdfa8ed7 100644 --- a/src/mono/mono/metadata/assembly-internals.h +++ b/src/mono/mono/metadata/assembly-internals.h @@ -142,10 +142,9 @@ void mono_set_assemblies_path_direct (char **path); typedef enum { - BUNDLED_BLOB, + BUNDLED_DATA, BUNDLED_ASSEMBLY, BUNDLED_SATELLITE_ASSEMBLY, - BUNDLED_BINARY, BUNDLED_PDB, BUNDLED_RESOURCE_COUNT, } BundledResourceType; diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 8ca0ce57a0f774..53d4dd9a1d05db 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -148,7 +148,7 @@ private static byte[] InitLookupTable() public static void GenerateRegisteredBundledObjects(string newFunctionName, ICollection<(string registeredName, string symbol, string type)> files, StreamWriter outputUtf8Writer) { - outputUtf8Writer.WriteLine("typedef enum {\n BUNDLED_BLOB,\n BUNDLED_ASSEMBLY,\n BUNDLED_SATELLITE_ASSEMBLY,\n BUNDLED_BINARY,\n BUNDLED_RESOURCE_COUNT,\n} BundledResourceType;"); + outputUtf8Writer.WriteLine("typedef enum {\n BUNDLED_DATA,\n BUNDLED_ASSEMBLY,\n BUNDLED_SATELLITE_ASSEMBLY,\n BUNDLED_PDB,\n BUNDLED_RESOURCE_COUNT,\n} BundledResourceType;"); outputUtf8Writer.WriteLine($"void mono_add_bundled_resource(const char *name, const char *culture, const unsigned char *data, unsigned int size, BundledResourceType type);"); outputUtf8Writer.WriteLine($"void mono_register_bundled_resources (void);"); outputUtf8Writer.WriteLine(); @@ -239,11 +239,11 @@ private static string GetFileType(string destinationFileName) { return "BUNDLED_ASSEMBLY"; } - if (destinationFileName.EndsWith(".bin")) + if (destinationFileName.EndsWith(".pdb")) { - return "BUNDLED_BINARY"; + return "BUNDLED_PDB"; } - return "BUNDLED_BLOB"; + return "BUNDLED_DATA"; } // Equivalent to "isalnum" From d7c09d0dea895f7ce3a4dafccca190d8dfd3ab56 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 11 Apr 2023 22:18:47 -0400 Subject: [PATCH 015/110] Add api to free memory dynamically allocated for the bundled resources hash table --- src/mono/mono/metadata/assembly.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index e31dd2ff81740e..acaee7a83bfeba 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -3208,6 +3208,24 @@ static GHashTable *bundled_resources = NULL; static int *resources_bundled_counts = NULL; +//--------------------------------------------------------------------------------------- +// +// mono_free_bundled_resources frees all memory allocated for bundled resources. +// It should only be called when the runtime no longer needs access to the data, +// most likely to happen during runtime shutdown. +// + +void +mono_free_bundled_resources () +{ + if (bundled_resources) + g_hash_table_destroy (bundled_resources); + bundled_resources = NULL; + + g_free (resources_bundled_counts); + resources_bundled_counts = NULL; +} + //--------------------------------------------------------------------------------------- // // mono_add_bundled_resource handles bundling of many types of resources to circumvent From d1b5f02beb7ca365259e0a30196789f956336a61 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 11 Apr 2023 22:24:34 -0400 Subject: [PATCH 016/110] Prepend enum and entries with mono --- src/mono/mono/metadata/assembly-internals.h | 16 +++++----- src/mono/mono/metadata/assembly.c | 30 +++++++++---------- src/tasks/EmitBundleTask/EmitBundleBase.cs | 12 ++++---- .../Templates/library-builder.h | 4 +-- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/mono/mono/metadata/assembly-internals.h b/src/mono/mono/metadata/assembly-internals.h index 96be75fdfa8ed7..0e48fc23e6ca48 100644 --- a/src/mono/mono/metadata/assembly-internals.h +++ b/src/mono/mono/metadata/assembly-internals.h @@ -142,22 +142,22 @@ void mono_set_assemblies_path_direct (char **path); typedef enum { - BUNDLED_DATA, - BUNDLED_ASSEMBLY, - BUNDLED_SATELLITE_ASSEMBLY, - BUNDLED_PDB, - BUNDLED_RESOURCE_COUNT, -} BundledResourceType; + MONO_BUNDLED_DATA, + MONO_BUNDLED_ASSEMBLY, + MONO_BUNDLED_SATELLITE_ASSEMBLY, + MONO_BUNDLED_PDB, + MONO_BUNDLED_RESOURCE_COUNT, +} MonoBundledResourceType; typedef struct BundledResource { const char *culture; // Satellite assemblies const unsigned char *data; unsigned int size; - BundledResourceType type; + MonoBundledResourceType type; } BundledResource; void -mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, BundledResourceType type); +mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, MonoBundledResourceType type); void mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size); diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index acaee7a83bfeba..29dca6e985ce52 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -3231,7 +3231,7 @@ mono_free_bundled_resources () // mono_add_bundled_resource handles bundling of many types of resources to circumvent // needing to find or have those resources on disk. The BundledResource struct models // the union of information carried by all supported types of resources which are -// enumerated in BundledResourceType. +// enumerated in MonoBundledResourceType. // // bundled_resources: // A single hash table will hold all resources being bundled with the understanding @@ -3253,9 +3253,9 @@ mono_free_bundled_resources () // void -mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, BundledResourceType type) +mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, MonoBundledResourceType type) { - if (type == BUNDLED_PDB) + if (type == MONO_BUNDLED_PDB) { mono_register_symfile_for_assembly (name, data, size); return; @@ -3265,7 +3265,7 @@ mono_add_bundled_resource (const char *name, const char *culture, const unsigned bundled_resources = g_hash_table_new (g_str_hash, g_str_equal); if (!resources_bundled_counts) - resources_bundled_counts = g_malloc0 (BUNDLED_RESOURCE_COUNT * sizeof (int)); + resources_bundled_counts = g_malloc0 (MONO_BUNDLED_RESOURCE_COUNT * sizeof (int)); g_assert (!g_hash_table_contains (bundled_resources, name)); @@ -3316,12 +3316,12 @@ populate_bundled_assemblies (gpointer key, gpointer value, gpointer user_data) { MonoBundledAssembly **bundle = (MonoBundledAssembly **)user_data; BundledResource *bundled_resource = (BundledResource *)value; - if (!bundle || !bundled_resource || bundled_resource->type != BUNDLED_ASSEMBLY) + if (!bundle || !bundled_resource || bundled_resource->type != MONO_BUNDLED_ASSEMBLY) return; MonoBundledAssembly *bundled_assembly = mono_create_new_bundled_assembly ((const char *)key, bundled_resource->data, bundled_resource->size); - g_assert (num_bundled_resources_to_register < resources_bundled_counts[BUNDLED_ASSEMBLY]); + g_assert (num_bundled_resources_to_register < resources_bundled_counts[MONO_BUNDLED_ASSEMBLY]); bundle [num_bundled_resources_to_register++] = bundled_assembly; } @@ -3336,19 +3336,19 @@ populate_bundled_satellite_assemblies (gpointer key, gpointer value, gpointer us { MonoBundledSatelliteAssembly **bundle = (MonoBundledSatelliteAssembly **)user_data; BundledResource *bundled_resource = (BundledResource *)value; - if (!bundle || !bundled_resource || bundled_resource->type != BUNDLED_SATELLITE_ASSEMBLY) + if (!bundle || !bundled_resource || bundled_resource->type != MONO_BUNDLED_SATELLITE_ASSEMBLY) return; MonoBundledSatelliteAssembly *bundled_satellite_assembly = mono_create_new_bundled_satellite_assembly ((const char *)key, bundled_resource->culture, bundled_resource->data, bundled_resource->size); - g_assert (num_bundled_resources_to_register < resources_bundled_counts[BUNDLED_SATELLITE_ASSEMBLY]); + g_assert (num_bundled_resources_to_register < resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY]); bundle [num_bundled_resources_to_register++] = bundled_satellite_assembly; } //--------------------------------------------------------------------------------------- // // mono_register_bundled_resources registers resources added by mono_add_bundled_resource. -// Out of the supported resource types in BundledResourceType, assemblies and satellite +// Out of the supported resource types in MonoBundledResourceType, assemblies and satellite // assemblies are registered. // // To appropriately allocate memory for the MonoBundledAssembly array and @@ -3364,23 +3364,23 @@ mono_register_bundled_resources (void) return; // register assemblies - if (resources_bundled_counts [BUNDLED_ASSEMBLY] != 0) + if (resources_bundled_counts [MONO_BUNDLED_ASSEMBLY] != 0) { - MonoBundledAssembly **assembly_bundle = g_new0 (MonoBundledAssembly*, resources_bundled_counts[BUNDLED_ASSEMBLY] + 1); + MonoBundledAssembly **assembly_bundle = g_new0 (MonoBundledAssembly*, resources_bundled_counts[MONO_BUNDLED_ASSEMBLY] + 1); num_bundled_resources_to_register = 0; g_hash_table_foreach (bundled_resources, populate_bundled_assemblies, assembly_bundle); - g_assert (num_bundled_resources_to_register == resources_bundled_counts[BUNDLED_ASSEMBLY]); + g_assert (num_bundled_resources_to_register == resources_bundled_counts[MONO_BUNDLED_ASSEMBLY]); mono_register_bundled_assemblies ((const MonoBundledAssembly **)assembly_bundle); } // register satellite assemblies - if (resources_bundled_counts [BUNDLED_SATELLITE_ASSEMBLY] != 0) + if (resources_bundled_counts [MONO_BUNDLED_SATELLITE_ASSEMBLY] != 0) { - MonoBundledSatelliteAssembly **satellite_assembly_bundle = g_new0 (MonoBundledSatelliteAssembly *, resources_bundled_counts[BUNDLED_SATELLITE_ASSEMBLY] + 1); + MonoBundledSatelliteAssembly **satellite_assembly_bundle = g_new0 (MonoBundledSatelliteAssembly *, resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY] + 1); num_bundled_resources_to_register = 0; g_hash_table_foreach (bundled_resources, populate_bundled_satellite_assemblies, satellite_assembly_bundle); - g_assert (num_bundled_resources_to_register == resources_bundled_counts[BUNDLED_SATELLITE_ASSEMBLY]); + g_assert (num_bundled_resources_to_register == resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY]); mono_register_bundled_satellite_assemblies ((const MonoBundledSatelliteAssembly **)satellite_assembly_bundle); } diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 53d4dd9a1d05db..a1f0353cbd4b90 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -148,8 +148,8 @@ private static byte[] InitLookupTable() public static void GenerateRegisteredBundledObjects(string newFunctionName, ICollection<(string registeredName, string symbol, string type)> files, StreamWriter outputUtf8Writer) { - outputUtf8Writer.WriteLine("typedef enum {\n BUNDLED_DATA,\n BUNDLED_ASSEMBLY,\n BUNDLED_SATELLITE_ASSEMBLY,\n BUNDLED_PDB,\n BUNDLED_RESOURCE_COUNT,\n} BundledResourceType;"); - outputUtf8Writer.WriteLine($"void mono_add_bundled_resource(const char *name, const char *culture, const unsigned char *data, unsigned int size, BundledResourceType type);"); + outputUtf8Writer.WriteLine("typedef enum {\n MONO_BUNDLED_DATA,\n MONO_BUNDLED_ASSEMBLY,\n MONO_BUNDLED_SATELLITE_ASSEMBLY,\n MONO_BUNDLED_PDB,\n MONO_BUNDLED_RESOURCE_COUNT,\n} MonoBundledResourceType;"); + outputUtf8Writer.WriteLine($"void mono_add_bundled_resource(const char *name, const char *culture, const unsigned char *data, unsigned int size, MonoBundledResourceType type);"); outputUtf8Writer.WriteLine($"void mono_register_bundled_resources (void);"); outputUtf8Writer.WriteLine(); @@ -233,17 +233,17 @@ private static string GetFileType(string destinationFileName) { if (destinationFileName.EndsWith(".resources.dll")) { - return "BUNDLED_SATELLITE_ASSEMBLY"; + return "MONO_BUNDLED_SATELLITE_ASSEMBLY"; } if (destinationFileName.EndsWith(".dll")) { - return "BUNDLED_ASSEMBLY"; + return "MONO_BUNDLED_ASSEMBLY"; } if (destinationFileName.EndsWith(".pdb")) { - return "BUNDLED_PDB"; + return "MONO_BUNDLED_PDB"; } - return "BUNDLED_DATA"; + return "MONO_BUNDLED_DATA"; } // Equivalent to "isalnum" diff --git a/src/tasks/LibraryBuilder/Templates/library-builder.h b/src/tasks/LibraryBuilder/Templates/library-builder.h index 0f3065f497e26b..5e7c2858542d7f 100644 --- a/src/tasks/LibraryBuilder/Templates/library-builder.h +++ b/src/tasks/LibraryBuilder/Templates/library-builder.h @@ -44,10 +44,10 @@ #endif // Platform specific native functions void register_aot_modules (void); -#if defined(BUNDLED_ASSEMBLIES) +#if defined(BUNDLED_RESOURCES) void mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size); void mono_register_resources_bundle (void); -#endif // BUNDLED_ASSEMBLIES +#endif // BUNDLED_RESOURCES void preload_assemblies_with_exported_symbols (); typedef void (*MonoRuntimeInitCallback) (void); void mono_set_runtime_init_callback (MonoRuntimeInitCallback callback); From c0c443c111b4eaeb2abed772563986b30db298cf Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Wed, 12 Apr 2023 00:32:32 -0400 Subject: [PATCH 017/110] Move bundling api into separate source and header --- src/mono/mono/metadata/CMakeLists.txt | 2 + src/mono/mono/metadata/assembly-internals.h | 27 +-- src/mono/mono/metadata/assembly.c | 184 +---------------- .../metadata/bundled-resources-internal.h | 35 ++++ src/mono/mono/metadata/bundled-resources.c | 193 ++++++++++++++++++ 5 files changed, 234 insertions(+), 207 deletions(-) create mode 100644 src/mono/mono/metadata/bundled-resources-internal.h create mode 100644 src/mono/mono/metadata/bundled-resources.c diff --git a/src/mono/mono/metadata/CMakeLists.txt b/src/mono/mono/metadata/CMakeLists.txt index abddf1a58a5d92..794c744ca87f5b 100644 --- a/src/mono/mono/metadata/CMakeLists.txt +++ b/src/mono/mono/metadata/CMakeLists.txt @@ -48,6 +48,8 @@ set(metadata_common_sources appdomain-icalls.h assembly.c assembly-internals.h + bundled-resources.c + bundled-resources-internal.h cil-coff.h class.c class-getters.h diff --git a/src/mono/mono/metadata/assembly-internals.h b/src/mono/mono/metadata/assembly-internals.h index 0e48fc23e6ca48..f76a28fa283236 100644 --- a/src/mono/mono/metadata/assembly-internals.h +++ b/src/mono/mono/metadata/assembly-internals.h @@ -138,32 +138,11 @@ mono_assembly_get_name_internal (MonoAssembly *assembly); MONO_PROFILER_API MonoImage* mono_assembly_get_image_internal (MonoAssembly *assembly); -void -mono_set_assemblies_path_direct (char **path); - -typedef enum { - MONO_BUNDLED_DATA, - MONO_BUNDLED_ASSEMBLY, - MONO_BUNDLED_SATELLITE_ASSEMBLY, - MONO_BUNDLED_PDB, - MONO_BUNDLED_RESOURCE_COUNT, -} MonoBundledResourceType; - -typedef struct BundledResource { - const char *culture; // Satellite assemblies - const unsigned char *data; - unsigned int size; - MonoBundledResourceType type; -} BundledResource; +MonoBundledAssembly * +mono_create_new_bundled_assembly (const char *name, const unsigned char *data, unsigned int size); void -mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, MonoBundledResourceType type); - -void -mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size); - -void -mono_register_bundled_resources (void); +mono_set_assemblies_path_direct (char **path); MONO_COMPONENT_API gboolean diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 29dca6e985ce52..07717dbf9251db 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -3159,7 +3159,7 @@ mono_assembly_get_name_internal (MonoAssembly *assembly) /** * mono_create_new_bundled_assembly: */ -static MonoBundledAssembly * +MonoBundledAssembly * mono_create_new_bundled_assembly (const char *name, const unsigned char *data, unsigned int size) { MonoBundledAssembly *assembly = g_new0 (MonoBundledAssembly, 1); @@ -3204,188 +3204,6 @@ mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly * satellite_bundles = assemblies; } -static GHashTable *bundled_resources = NULL; - -static int *resources_bundled_counts = NULL; - -//--------------------------------------------------------------------------------------- -// -// mono_free_bundled_resources frees all memory allocated for bundled resources. -// It should only be called when the runtime no longer needs access to the data, -// most likely to happen during runtime shutdown. -// - -void -mono_free_bundled_resources () -{ - if (bundled_resources) - g_hash_table_destroy (bundled_resources); - bundled_resources = NULL; - - g_free (resources_bundled_counts); - resources_bundled_counts = NULL; -} - -//--------------------------------------------------------------------------------------- -// -// mono_add_bundled_resource handles bundling of many types of resources to circumvent -// needing to find or have those resources on disk. The BundledResource struct models -// the union of information carried by all supported types of resources which are -// enumerated in MonoBundledResourceType. -// -// bundled_resources: -// A single hash table will hold all resources being bundled with the understanding -// that all resources being bundled are uniquely named. The bundled resource is tagged -// with the type of resource it represents for later processing of specific types of -// resources as done in mono_register_bundled_resources. -// -// resources_bundled_counts: -// An integer array tracks the number of each resource type added to the bundle -// to facilitate appropriate memory allocation when processing the bundle as done in -// mono_register_bundled_resources -// -// Arguments: -// * name - Unique name of the resource -// * culture - Culture of satellite assemblies -// * data - Byte data array of the bundled resource -// * size - Size of byte data array of the bundled resource -// type - Type of bundled resource -// - -void -mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, MonoBundledResourceType type) -{ - if (type == MONO_BUNDLED_PDB) - { - mono_register_symfile_for_assembly (name, data, size); - return; - } - - if (!bundled_resources) - bundled_resources = g_hash_table_new (g_str_hash, g_str_equal); - - if (!resources_bundled_counts) - resources_bundled_counts = g_malloc0 (MONO_BUNDLED_RESOURCE_COUNT * sizeof (int)); - - g_assert (!g_hash_table_contains (bundled_resources, name)); - - BundledResource *bundled_resource = g_new0 (BundledResource, 1); - bundled_resource->culture = culture; - bundled_resource->data = data; - bundled_resource->size = size; - bundled_resource->type = type; - ++resources_bundled_counts[type]; - g_hash_table_insert (bundled_resources, (gpointer) name, bundled_resource); -} - -//--------------------------------------------------------------------------------------- -// -// mono_get_bundled_resource_data retrieves the data of the bundled resource. If the -// requested bundled resource's name has been added via mono_add_bundled_resource, -// the byte array data and data size are populated. -// -// Arguments: -// * name - Unique name of the resource -// ** out_data - the pointer to the byte array data -// ** out_size - the pointer to the data size - -void -mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size) -{ - *out_data = NULL; - *out_size = 0; - - BundledResource *bundled_resource; - - if (g_hash_table_lookup_extended (bundled_resources, name, NULL, (gpointer *)&bundled_resource)) { - *out_data = bundled_resource->data; - *out_size = bundled_resource->size; - } -} - -static int num_bundled_resources_to_register = 0; - -//--------------------------------------------------------------------------------------- -// -// populate_bundled_assemblies assists in populating the MonoBundledAssembly -// to register in mono_register_bundled_resources -// - -static void -populate_bundled_assemblies (gpointer key, gpointer value, gpointer user_data) -{ - MonoBundledAssembly **bundle = (MonoBundledAssembly **)user_data; - BundledResource *bundled_resource = (BundledResource *)value; - if (!bundle || !bundled_resource || bundled_resource->type != MONO_BUNDLED_ASSEMBLY) - return; - - MonoBundledAssembly *bundled_assembly = mono_create_new_bundled_assembly ((const char *)key, bundled_resource->data, bundled_resource->size); - - g_assert (num_bundled_resources_to_register < resources_bundled_counts[MONO_BUNDLED_ASSEMBLY]); - bundle [num_bundled_resources_to_register++] = bundled_assembly; -} - -//--------------------------------------------------------------------------------------- -// -// populate_bundled_satellite_assemblies assists in populating the MonoBundledSatelliteAssembly -// to register in mono_register_bundled_resources -// - -static void -populate_bundled_satellite_assemblies (gpointer key, gpointer value, gpointer user_data) -{ - MonoBundledSatelliteAssembly **bundle = (MonoBundledSatelliteAssembly **)user_data; - BundledResource *bundled_resource = (BundledResource *)value; - if (!bundle || !bundled_resource || bundled_resource->type != MONO_BUNDLED_SATELLITE_ASSEMBLY) - return; - - MonoBundledSatelliteAssembly *bundled_satellite_assembly = mono_create_new_bundled_satellite_assembly ((const char *)key, bundled_resource->culture, bundled_resource->data, bundled_resource->size); - - g_assert (num_bundled_resources_to_register < resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY]); - bundle [num_bundled_resources_to_register++] = bundled_satellite_assembly; -} - -//--------------------------------------------------------------------------------------- -// -// mono_register_bundled_resources registers resources added by mono_add_bundled_resource. -// Out of the supported resource types in MonoBundledResourceType, assemblies and satellite -// assemblies are registered. -// -// To appropriately allocate memory for the MonoBundledAssembly array and -// MonoBundledSatelliteAssembly array to register, resources_bundled_counts is leveraged, -// and therefore, mono_add_bundled_resource must be previously invoked for any successful -// registering of assemblies and satellite assemblies. -// - -void -mono_register_bundled_resources (void) -{ - if (!bundled_resources) - return; - - // register assemblies - if (resources_bundled_counts [MONO_BUNDLED_ASSEMBLY] != 0) - { - MonoBundledAssembly **assembly_bundle = g_new0 (MonoBundledAssembly*, resources_bundled_counts[MONO_BUNDLED_ASSEMBLY] + 1); - num_bundled_resources_to_register = 0; - g_hash_table_foreach (bundled_resources, populate_bundled_assemblies, assembly_bundle); - g_assert (num_bundled_resources_to_register == resources_bundled_counts[MONO_BUNDLED_ASSEMBLY]); - - mono_register_bundled_assemblies ((const MonoBundledAssembly **)assembly_bundle); - } - - // register satellite assemblies - if (resources_bundled_counts [MONO_BUNDLED_SATELLITE_ASSEMBLY] != 0) - { - MonoBundledSatelliteAssembly **satellite_assembly_bundle = g_new0 (MonoBundledSatelliteAssembly *, resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY] + 1); - num_bundled_resources_to_register = 0; - g_hash_table_foreach (bundled_resources, populate_bundled_satellite_assemblies, satellite_assembly_bundle); - g_assert (num_bundled_resources_to_register == resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY]); - - mono_register_bundled_satellite_assemblies ((const MonoBundledSatelliteAssembly **)satellite_assembly_bundle); - } -} - /** * mono_assembly_is_jit_optimizer_disabled: * diff --git a/src/mono/mono/metadata/bundled-resources-internal.h b/src/mono/mono/metadata/bundled-resources-internal.h new file mode 100644 index 00000000000000..9dd2ddabd6cceb --- /dev/null +++ b/src/mono/mono/metadata/bundled-resources-internal.h @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +#ifndef __MONO_METADATA_BUNDLED_RESOURCES_H__ +#define __MONO_METADATA_BUNDLED_RESOURCES_H__ + +typedef enum { + MONO_BUNDLED_DATA, + MONO_BUNDLED_ASSEMBLY, + MONO_BUNDLED_SATELLITE_ASSEMBLY, + MONO_BUNDLED_PDB, + MONO_BUNDLED_RESOURCE_COUNT, +} MonoBundledResourceType; + +typedef struct BundledResource { + const char *culture; // Satellite assemblies + const unsigned char *data; + unsigned int size; + MonoBundledResourceType type; +} BundledResource; + +void +mono_free_bundled_resources (void); + +void +mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, MonoBundledResourceType type); + +void +mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size); + +void +mono_register_bundled_resources (void); + +#endif /* __MONO_METADATA_BUNDLED_RESOURCES_H__ */ \ No newline at end of file diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c new file mode 100644 index 00000000000000..117ab76f3df634 --- /dev/null +++ b/src/mono/mono/metadata/bundled-resources.c @@ -0,0 +1,193 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +#include + +#include "bundled-resources-internal.h" +#include "assembly-internals.h" + +#include +#include + +static GHashTable *bundled_resources = NULL; + +static int *resources_bundled_counts = NULL; + +//--------------------------------------------------------------------------------------- +// +// mono_free_bundled_resources frees all memory allocated for bundled resources. +// It should only be called when the runtime no longer needs access to the data, +// most likely to happen during runtime shutdown. +// + +void +mono_free_bundled_resources (void) +{ + if (bundled_resources) + g_hash_table_destroy (bundled_resources); + bundled_resources = NULL; + + g_free (resources_bundled_counts); + resources_bundled_counts = NULL; +} + +//--------------------------------------------------------------------------------------- +// +// mono_add_bundled_resource handles bundling of many types of resources to circumvent +// needing to find or have those resources on disk. The BundledResource struct models +// the union of information carried by all supported types of resources which are +// enumerated in MonoBundledResourceType. +// +// bundled_resources: +// A single hash table will hold all resources being bundled with the understanding +// that all resources being bundled are uniquely named. The bundled resource is tagged +// with the type of resource it represents for later processing of specific types of +// resources as done in mono_register_bundled_resources. +// +// resources_bundled_counts: +// An integer array tracks the number of each resource type added to the bundle +// to facilitate appropriate memory allocation when processing the bundle as done in +// mono_register_bundled_resources +// +// Arguments: +// * name - Unique name of the resource +// * culture - Culture of satellite assemblies +// * data - Byte data array of the bundled resource +// * size - Size of byte data array of the bundled resource +// type - Type of bundled resource +// + +void +mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, MonoBundledResourceType type) +{ + if (type == MONO_BUNDLED_PDB) + { + mono_register_symfile_for_assembly (name, data, size); + return; + } + + if (!bundled_resources) + bundled_resources = g_hash_table_new (g_str_hash, g_str_equal); + + if (!resources_bundled_counts) + resources_bundled_counts = g_malloc0 (MONO_BUNDLED_RESOURCE_COUNT * sizeof (int)); + + g_assert (!g_hash_table_contains (bundled_resources, name)); + + BundledResource *bundled_resource = g_new0 (BundledResource, 1); + bundled_resource->culture = culture; + bundled_resource->data = data; + bundled_resource->size = size; + bundled_resource->type = type; + ++resources_bundled_counts[type]; + g_hash_table_insert (bundled_resources, (gpointer) name, bundled_resource); +} + +//--------------------------------------------------------------------------------------- +// +// mono_get_bundled_resource_data retrieves the data of the bundled resource. If the +// requested bundled resource's name has been added via mono_add_bundled_resource, +// the byte array data and data size are populated. +// +// Arguments: +// * name - Unique name of the resource +// ** out_data - the pointer to the byte array data +// ** out_size - the pointer to the data size + +void +mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size) +{ + *out_data = NULL; + *out_size = 0; + + BundledResource *bundled_resource; + + if (g_hash_table_lookup_extended (bundled_resources, name, NULL, (gpointer *)&bundled_resource)) { + *out_data = bundled_resource->data; + *out_size = bundled_resource->size; + } +} + +static int num_bundled_resources_to_register = 0; + +//--------------------------------------------------------------------------------------- +// +// populate_bundled_assemblies assists in populating the MonoBundledAssembly +// to register in mono_register_bundled_resources +// + +static void +populate_bundled_assemblies (gpointer key, gpointer value, gpointer user_data) +{ + MonoBundledAssembly **bundle = (MonoBundledAssembly **)user_data; + BundledResource *bundled_resource = (BundledResource *)value; + if (!bundle || !bundled_resource || bundled_resource->type != MONO_BUNDLED_ASSEMBLY) + return; + + MonoBundledAssembly *bundled_assembly = mono_create_new_bundled_assembly ((const char *)key, bundled_resource->data, bundled_resource->size); + + g_assert (num_bundled_resources_to_register < resources_bundled_counts[MONO_BUNDLED_ASSEMBLY]); + bundle [num_bundled_resources_to_register++] = bundled_assembly; +} + +//--------------------------------------------------------------------------------------- +// +// populate_bundled_satellite_assemblies assists in populating the MonoBundledSatelliteAssembly +// to register in mono_register_bundled_resources +// + +static void +populate_bundled_satellite_assemblies (gpointer key, gpointer value, gpointer user_data) +{ + MonoBundledSatelliteAssembly **bundle = (MonoBundledSatelliteAssembly **)user_data; + BundledResource *bundled_resource = (BundledResource *)value; + if (!bundle || !bundled_resource || bundled_resource->type != MONO_BUNDLED_SATELLITE_ASSEMBLY) + return; + + MonoBundledSatelliteAssembly *bundled_satellite_assembly = mono_create_new_bundled_satellite_assembly ((const char *)key, bundled_resource->culture, bundled_resource->data, bundled_resource->size); + + g_assert (num_bundled_resources_to_register < resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY]); + bundle [num_bundled_resources_to_register++] = bundled_satellite_assembly; +} + +//--------------------------------------------------------------------------------------- +// +// mono_register_bundled_resources registers resources added by mono_add_bundled_resource. +// Out of the supported resource types in MonoBundledResourceType, assemblies and satellite +// assemblies are registered. +// +// To appropriately allocate memory for the MonoBundledAssembly array and +// MonoBundledSatelliteAssembly array to register, resources_bundled_counts is leveraged, +// and therefore, mono_add_bundled_resource must be previously invoked for any successful +// registering of assemblies and satellite assemblies. +// + +void +mono_register_bundled_resources (void) +{ + if (!bundled_resources) + return; + + // register assemblies + if (resources_bundled_counts [MONO_BUNDLED_ASSEMBLY] != 0) + { + MonoBundledAssembly **assembly_bundle = g_new0 (MonoBundledAssembly*, resources_bundled_counts[MONO_BUNDLED_ASSEMBLY] + 1); + num_bundled_resources_to_register = 0; + g_hash_table_foreach (bundled_resources, populate_bundled_assemblies, assembly_bundle); + g_assert (num_bundled_resources_to_register == resources_bundled_counts[MONO_BUNDLED_ASSEMBLY]); + + mono_register_bundled_assemblies ((const MonoBundledAssembly **)assembly_bundle); + } + + // register satellite assemblies + if (resources_bundled_counts [MONO_BUNDLED_SATELLITE_ASSEMBLY] != 0) + { + MonoBundledSatelliteAssembly **satellite_assembly_bundle = g_new0 (MonoBundledSatelliteAssembly *, resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY] + 1); + num_bundled_resources_to_register = 0; + g_hash_table_foreach (bundled_resources, populate_bundled_satellite_assemblies, satellite_assembly_bundle); + g_assert (num_bundled_resources_to_register == resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY]); + + mono_register_bundled_satellite_assemblies ((const MonoBundledSatelliteAssembly **)satellite_assembly_bundle); + } +} From 7683ed15a8e31452c6b8b8ee5cebe8f0f27895d4 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 18 Apr 2023 10:59:24 -0400 Subject: [PATCH 018/110] Fix source file name --- src/mono/mono/metadata/CMakeLists.txt | 2 +- ...d-resources-internal.h => bundled-resources-internals.h} | 6 +++--- src/mono/mono/metadata/bundled-resources.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/mono/mono/metadata/{bundled-resources-internal.h => bundled-resources-internals.h} (83%) diff --git a/src/mono/mono/metadata/CMakeLists.txt b/src/mono/mono/metadata/CMakeLists.txt index 794c744ca87f5b..57b0ffa280282c 100644 --- a/src/mono/mono/metadata/CMakeLists.txt +++ b/src/mono/mono/metadata/CMakeLists.txt @@ -49,7 +49,7 @@ set(metadata_common_sources assembly.c assembly-internals.h bundled-resources.c - bundled-resources-internal.h + bundled-resources-internals.h cil-coff.h class.c class-getters.h diff --git a/src/mono/mono/metadata/bundled-resources-internal.h b/src/mono/mono/metadata/bundled-resources-internals.h similarity index 83% rename from src/mono/mono/metadata/bundled-resources-internal.h rename to src/mono/mono/metadata/bundled-resources-internals.h index 9dd2ddabd6cceb..45346671373f4d 100644 --- a/src/mono/mono/metadata/bundled-resources-internal.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -2,8 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. // -#ifndef __MONO_METADATA_BUNDLED_RESOURCES_H__ -#define __MONO_METADATA_BUNDLED_RESOURCES_H__ +#ifndef __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ +#define __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ typedef enum { MONO_BUNDLED_DATA, @@ -32,4 +32,4 @@ mono_get_bundled_resource_data (const char *name, const unsigned char **out_data void mono_register_bundled_resources (void); -#endif /* __MONO_METADATA_BUNDLED_RESOURCES_H__ */ \ No newline at end of file +#endif /* __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ */ \ No newline at end of file diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 117ab76f3df634..e57d83632c6bda 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -4,7 +4,7 @@ #include -#include "bundled-resources-internal.h" +#include "bundled-resources-internals.h" #include "assembly-internals.h" #include From b1f3240eee966a39f804710b44d715bef9dec27f Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 18 Apr 2023 11:00:05 -0400 Subject: [PATCH 019/110] Prefix struct with Mono --- src/mono/mono/metadata/bundled-resources-internals.h | 4 ++-- src/mono/mono/metadata/bundled-resources.c | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 45346671373f4d..1447be640ea819 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -13,12 +13,12 @@ typedef enum { MONO_BUNDLED_RESOURCE_COUNT, } MonoBundledResourceType; -typedef struct BundledResource { +typedef struct MonoBundledResource { const char *culture; // Satellite assemblies const unsigned char *data; unsigned int size; MonoBundledResourceType type; -} BundledResource; +} MonoBundledResource; void mono_free_bundled_resources (void); diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index e57d83632c6bda..1c8c0762b14b12 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -35,7 +35,7 @@ mono_free_bundled_resources (void) //--------------------------------------------------------------------------------------- // // mono_add_bundled_resource handles bundling of many types of resources to circumvent -// needing to find or have those resources on disk. The BundledResource struct models +// needing to find or have those resources on disk. The MonoBundledResource struct models // the union of information carried by all supported types of resources which are // enumerated in MonoBundledResourceType. // @@ -75,7 +75,7 @@ mono_add_bundled_resource (const char *name, const char *culture, const unsigned g_assert (!g_hash_table_contains (bundled_resources, name)); - BundledResource *bundled_resource = g_new0 (BundledResource, 1); + MonoBundledResource *bundled_resource = g_new0 (MonoBundledResource, 1); bundled_resource->culture = culture; bundled_resource->data = data; bundled_resource->size = size; @@ -101,7 +101,7 @@ mono_get_bundled_resource_data (const char *name, const unsigned char **out_data *out_data = NULL; *out_size = 0; - BundledResource *bundled_resource; + MonoBundledResource *bundled_resource; if (g_hash_table_lookup_extended (bundled_resources, name, NULL, (gpointer *)&bundled_resource)) { *out_data = bundled_resource->data; @@ -121,7 +121,7 @@ static void populate_bundled_assemblies (gpointer key, gpointer value, gpointer user_data) { MonoBundledAssembly **bundle = (MonoBundledAssembly **)user_data; - BundledResource *bundled_resource = (BundledResource *)value; + MonoBundledResource *bundled_resource = (MonoBundledResource *)value; if (!bundle || !bundled_resource || bundled_resource->type != MONO_BUNDLED_ASSEMBLY) return; @@ -141,7 +141,7 @@ static void populate_bundled_satellite_assemblies (gpointer key, gpointer value, gpointer user_data) { MonoBundledSatelliteAssembly **bundle = (MonoBundledSatelliteAssembly **)user_data; - BundledResource *bundled_resource = (BundledResource *)value; + MonoBundledResource *bundled_resource = (MonoBundledResource *)value; if (!bundle || !bundled_resource || bundled_resource->type != MONO_BUNDLED_SATELLITE_ASSEMBLY) return; From ddf66913de2ad4a7c9caa3ad795ab72af8cce51d Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 18 Apr 2023 11:02:01 -0400 Subject: [PATCH 020/110] Integrate new bundling api into mono debug --- .../metadata/bundled-resources-internals.h | 2 +- src/mono/mono/metadata/bundled-resources.c | 6 ------ src/mono/mono/metadata/mono-debug.c | 21 +++++++++++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 1447be640ea819..e882768dbf7860 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -9,7 +9,7 @@ typedef enum { MONO_BUNDLED_DATA, MONO_BUNDLED_ASSEMBLY, MONO_BUNDLED_SATELLITE_ASSEMBLY, - MONO_BUNDLED_PDB, + MONO_BUNDLED_SYMFILE, MONO_BUNDLED_RESOURCE_COUNT, } MonoBundledResourceType; diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 1c8c0762b14b12..cf91bbeefb8e34 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -61,12 +61,6 @@ mono_free_bundled_resources (void) void mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, MonoBundledResourceType type) { - if (type == MONO_BUNDLED_PDB) - { - mono_register_symfile_for_assembly (name, data, size); - return; - } - if (!bundled_resources) bundled_resources = g_hash_table_new (g_str_hash, g_str_equal); diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 993a6fd1edfbec..8c9dfc5cf1ec0a 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1120,6 +1121,26 @@ open_symfile_from_bundle (MonoImage *image) { BundledSymfile *bsymfile; + const unsigned char *data; + unsigned int data_len; + mono_get_bundled_resource_data (image->module_name, &data, &data_len); + if (data) + return mono_debug_open_image (image, data, data_len); + +#ifdef ENABLE_WEBCIL + int len = strlen (image->module_name); + char *module_name_dll_suffix = strdup (image->module_name); + /* if image's module_name ends with .webcil, check if theres a bundled resource with a .dll extension instead */ + if (module_name_dll_suffix && !g_strcasecmp (".webcil", &image->module_name [len - 7])) { + memcpy (module_name_dll_suffix + len - 7, ".dll", 4); + *(module_name_dll_suffix + len - 3) = '\0'; + mono_get_bundled_resource_data (module_name_dll_suffix, &data, &data_len); + } + g_free (module_name_dll_suffix); + if (data) + return mono_debug_open_image (image, data, data_len); +#endif + for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { if (!bsymfile_match (bsymfile, image->module_name)) continue; From 130aff337d70e57b6eeb43b81a529789b1198038 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 18 Apr 2023 11:02:29 -0400 Subject: [PATCH 021/110] Fix types in autoinit --- src/tasks/LibraryBuilder/Templates/autoinit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index c88abd7fac6f58..74a4bee2bedf98 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -36,8 +36,8 @@ initialize_runtimeconfig (const char *bundle_path) LOG_ERROR ("Out of memory.\n"); const char *file_name = "runtimeconfig.bin"; - const char *data; - int data_len; + const unsigned char *data; + unsigned int data_len; mono_get_bundled_resource_data (file_name, &data, &data_len); if (data) { From b8177d0e90bcdbcc1946adc2f6b5097cb211b573 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 21 Apr 2023 20:32:59 -0400 Subject: [PATCH 022/110] [mono] Base bundling apis around preallocation of MonoBundled*Resource structs --- .../metadata/bundled-resources-internals.h | 49 ++++-- src/mono/mono/metadata/bundled-resources.c | 166 ++++-------------- src/tasks/EmitBundleTask/EmitBundleBase.cs | 4 - 3 files changed, 75 insertions(+), 144 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index e882768dbf7860..6cb1c1359f06d3 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -5,31 +5,58 @@ #ifndef __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ #define __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ +#include "loader-internals.h" + +#include +#include + typedef enum { MONO_BUNDLED_DATA, MONO_BUNDLED_ASSEMBLY, MONO_BUNDLED_SATELLITE_ASSEMBLY, - MONO_BUNDLED_SYMFILE, MONO_BUNDLED_RESOURCE_COUNT, } MonoBundledResourceType; -typedef struct MonoBundledResource { - const char *culture; // Satellite assemblies - const unsigned char *data; - unsigned int size; +typedef struct _MonoBundledResource { MonoBundledResourceType type; + void (*free_bundled_resource_func)(void *); } MonoBundledResource; -void -mono_free_bundled_resources (void); +typedef struct _MonoBundledData { + char *name; + const unsigned char *data; + unsigned int size; +} MonoBundledData; -void -mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, MonoBundledResourceType type); +typedef struct _MonoBundledDataResource { + MonoBundledResource resource; + MonoBundledData data; +} MonoBundledDataResource; + +typedef struct _MonoBundledSymfile { + const unsigned char *data; + unsigned int size; +} MonoBundledSymfile; + +typedef struct _MonoBundledAssemblyResource { + MonoBundledResource resource; + MonoBundledAssembly assembly; + MonoBundledSymfile symfile; +} MonoBundledAssemblyResource; + +typedef struct _MonoBundledSatelliteAssemblyResource { + MonoBundledResource resource; + MonoBundledSatelliteAssembly satellite_assembly; + MonoBundledSymfile symfile; +} MonoBundledSatelliteAssemblyResource; void -mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size); +mono_free_bundled_resources (void); void -mono_register_bundled_resources (void); +mono_add_bundled_resource (MonoBundledResource **resources_to_bundle, uint32_t len); + +MonoBundledResource * +mono_get_bundled_resource_data (const char *name); #endif /* __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ */ \ No newline at end of file diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index cf91bbeefb8e34..7064801b1fa8b4 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -12,8 +12,6 @@ static GHashTable *bundled_resources = NULL; -static int *resources_bundled_counts = NULL; - //--------------------------------------------------------------------------------------- // // mono_free_bundled_resources frees all memory allocated for bundled resources. @@ -24,12 +22,8 @@ static int *resources_bundled_counts = NULL; void mono_free_bundled_resources (void) { - if (bundled_resources) - g_hash_table_destroy (bundled_resources); + g_hash_table_destroy (bundled_resources); bundled_resources = NULL; - - g_free (resources_bundled_counts); - resources_bundled_counts = NULL; } //--------------------------------------------------------------------------------------- @@ -42,146 +36,60 @@ mono_free_bundled_resources (void) // bundled_resources: // A single hash table will hold all resources being bundled with the understanding // that all resources being bundled are uniquely named. The bundled resource is tagged -// with the type of resource it represents for later processing of specific types of -// resources as done in mono_register_bundled_resources. -// -// resources_bundled_counts: -// An integer array tracks the number of each resource type added to the bundle -// to facilitate appropriate memory allocation when processing the bundle as done in -// mono_register_bundled_resources +// with the type of resource it represents, and the pointer added to this hash table +// should fully represent a MonoBundled*Resource struct defined in `bundled-resources-internals.h // // Arguments: -// * name - Unique name of the resource -// * culture - Culture of satellite assemblies -// * data - Byte data array of the bundled resource -// * size - Size of byte data array of the bundled resource -// type - Type of bundled resource +// ** resources_to_bundle - An array of pointers to `MonoBundledResource`, which details +// the type of MonoBundled*Resource information follows this pointer in memory. +// len - The number of resources being added to the hash table // void -mono_add_bundled_resource (const char *name, const char *culture, const unsigned char *data, unsigned int size, MonoBundledResourceType type) +mono_add_bundled_resource (MonoBundledResource **resources_to_bundle, uint32_t len) { if (!bundled_resources) bundled_resources = g_hash_table_new (g_str_hash, g_str_equal); - if (!resources_bundled_counts) - resources_bundled_counts = g_malloc0 (MONO_BUNDLED_RESOURCE_COUNT * sizeof (int)); - - g_assert (!g_hash_table_contains (bundled_resources, name)); - - MonoBundledResource *bundled_resource = g_new0 (MonoBundledResource, 1); - bundled_resource->culture = culture; - bundled_resource->data = data; - bundled_resource->size = size; - bundled_resource->type = type; - ++resources_bundled_counts[type]; - g_hash_table_insert (bundled_resources, (gpointer) name, bundled_resource); + for (int i = 0; i < len; ++i) { + MonoBundledResource *resource_to_bundle = (MonoBundledResource *)resources_to_bundle[i]; + switch (resource_to_bundle->type) { + case MONO_BUNDLED_ASSEMBLY: { + MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)resource_to_bundle; + g_hash_table_insert (bundled_resources, (gpointer) assembly->assembly.name, assembly); + break; + } + case MONO_BUNDLED_SATELLITE_ASSEMBLY: { + MonoBundledSatelliteAssemblyResource *satellite_assembly = (MonoBundledSatelliteAssemblyResource *)resource_to_bundle; + g_hash_table_insert (bundled_resources, (gpointer) satellite_assembly->satellite_assembly.name, satellite_assembly); + break; + } + case MONO_BUNDLED_DATA: + default: { + MonoBundledDataResource *data = (MonoBundledDataResource *)resource_to_bundle; + g_hash_table_insert (bundled_resources, (gpointer) data->data.name, data); + break; + } + } + } } //--------------------------------------------------------------------------------------- // -// mono_get_bundled_resource_data retrieves the data of the bundled resource. If the -// requested bundled resource's name has been added via mono_add_bundled_resource, -// the byte array data and data size are populated. +// mono_get_bundled_resource_data retrieves the pointer of the MonoBundledResource associated +// with a key equivalent to the requested resource name. If the requested bundled resource's +// name has been added via mono_add_bundled_resource, a MonoBundled*Resource had been +// preallocated, typically through EmitBundleTask. // // Arguments: // * name - Unique name of the resource -// ** out_data - the pointer to the byte array data -// ** out_size - the pointer to the data size - -void -mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size) -{ - *out_data = NULL; - *out_size = 0; - - MonoBundledResource *bundled_resource; - - if (g_hash_table_lookup_extended (bundled_resources, name, NULL, (gpointer *)&bundled_resource)) { - *out_data = bundled_resource->data; - *out_size = bundled_resource->size; - } -} - -static int num_bundled_resources_to_register = 0; - -//--------------------------------------------------------------------------------------- -// -// populate_bundled_assemblies assists in populating the MonoBundledAssembly -// to register in mono_register_bundled_resources -// - -static void -populate_bundled_assemblies (gpointer key, gpointer value, gpointer user_data) -{ - MonoBundledAssembly **bundle = (MonoBundledAssembly **)user_data; - MonoBundledResource *bundled_resource = (MonoBundledResource *)value; - if (!bundle || !bundled_resource || bundled_resource->type != MONO_BUNDLED_ASSEMBLY) - return; - - MonoBundledAssembly *bundled_assembly = mono_create_new_bundled_assembly ((const char *)key, bundled_resource->data, bundled_resource->size); - - g_assert (num_bundled_resources_to_register < resources_bundled_counts[MONO_BUNDLED_ASSEMBLY]); - bundle [num_bundled_resources_to_register++] = bundled_assembly; -} - -//--------------------------------------------------------------------------------------- -// -// populate_bundled_satellite_assemblies assists in populating the MonoBundledSatelliteAssembly -// to register in mono_register_bundled_resources -// - -static void -populate_bundled_satellite_assemblies (gpointer key, gpointer value, gpointer user_data) -{ - MonoBundledSatelliteAssembly **bundle = (MonoBundledSatelliteAssembly **)user_data; - MonoBundledResource *bundled_resource = (MonoBundledResource *)value; - if (!bundle || !bundled_resource || bundled_resource->type != MONO_BUNDLED_SATELLITE_ASSEMBLY) - return; - - MonoBundledSatelliteAssembly *bundled_satellite_assembly = mono_create_new_bundled_satellite_assembly ((const char *)key, bundled_resource->culture, bundled_resource->data, bundled_resource->size); - - g_assert (num_bundled_resources_to_register < resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY]); - bundle [num_bundled_resources_to_register++] = bundled_satellite_assembly; -} - -//--------------------------------------------------------------------------------------- -// -// mono_register_bundled_resources registers resources added by mono_add_bundled_resource. -// Out of the supported resource types in MonoBundledResourceType, assemblies and satellite -// assemblies are registered. // -// To appropriately allocate memory for the MonoBundledAssembly array and -// MonoBundledSatelliteAssembly array to register, resources_bundled_counts is leveraged, -// and therefore, mono_add_bundled_resource must be previously invoked for any successful -// registering of assemblies and satellite assemblies. +// Returns: +// MonoBundledResource * - Pointer to the resource in the hashmap with the key `name` // -void -mono_register_bundled_resources (void) +MonoBundledResource * +mono_get_bundled_resource_data (const char *name) { - if (!bundled_resources) - return; - - // register assemblies - if (resources_bundled_counts [MONO_BUNDLED_ASSEMBLY] != 0) - { - MonoBundledAssembly **assembly_bundle = g_new0 (MonoBundledAssembly*, resources_bundled_counts[MONO_BUNDLED_ASSEMBLY] + 1); - num_bundled_resources_to_register = 0; - g_hash_table_foreach (bundled_resources, populate_bundled_assemblies, assembly_bundle); - g_assert (num_bundled_resources_to_register == resources_bundled_counts[MONO_BUNDLED_ASSEMBLY]); - - mono_register_bundled_assemblies ((const MonoBundledAssembly **)assembly_bundle); - } - - // register satellite assemblies - if (resources_bundled_counts [MONO_BUNDLED_SATELLITE_ASSEMBLY] != 0) - { - MonoBundledSatelliteAssembly **satellite_assembly_bundle = g_new0 (MonoBundledSatelliteAssembly *, resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY] + 1); - num_bundled_resources_to_register = 0; - g_hash_table_foreach (bundled_resources, populate_bundled_satellite_assemblies, satellite_assembly_bundle); - g_assert (num_bundled_resources_to_register == resources_bundled_counts[MONO_BUNDLED_SATELLITE_ASSEMBLY]); - - mono_register_bundled_satellite_assemblies ((const MonoBundledSatelliteAssembly **)satellite_assembly_bundle); - } + return g_hash_table_lookup (bundled_resources, name); } diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index a1f0353cbd4b90..5e8bebc74e0f0c 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -239,10 +239,6 @@ private static string GetFileType(string destinationFileName) { return "MONO_BUNDLED_ASSEMBLY"; } - if (destinationFileName.EndsWith(".pdb")) - { - return "MONO_BUNDLED_PDB"; - } return "MONO_BUNDLED_DATA"; } From 60aa11055f4a14b9e215e332cd6e1ec40a776b5b Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 21 Apr 2023 20:44:14 -0400 Subject: [PATCH 023/110] [mono] Clean up bundled assembly and satellite assembly search --- src/mono/mono/metadata/assembly.c | 40 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 07717dbf9251db..19645d35358f91 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1470,23 +1470,19 @@ bundled_assembly_match (const char *bundled_name, const char *name) } static MonoImage * -open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status, gboolean is_satellite) +open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status) { if (!bundles) return NULL; - MonoImage *image = NULL; - char *name = is_satellite ? g_strdup (filename) : g_path_get_basename (filename); - for (int i = 0; !image && bundles [i]; ++i) { - if (bundled_assembly_match (bundles[i]->name, name)) { + for (int i = 0; bundles [i]; ++i) { + if (bundled_assembly_match (bundles[i]->name, filename)) { // Since bundled images don't exist on disk, don't give them a legit filename - image = mono_image_open_from_data_internal (alc, (char*)bundles [i]->data, bundles [i]->size, FALSE, status, FALSE, name, NULL); - break; + return mono_image_open_from_data_internal (alc, (char*)bundles [i]->data, bundles [i]->size, FALSE, status, FALSE, filename, NULL); } } - g_free (name); - return image; + return NULL; } static MonoImage * @@ -1496,18 +1492,16 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, return NULL; MonoImage *image = NULL; - char *name = g_strdup (filename); + char *bundle_name = g_strconcat (culture, "/", filename, (const char *)NULL); for (int i = 0; !image && satellite_bundles [i]; ++i) { - if (bundled_assembly_match (satellite_bundles[i]->name, name) && strcmp (satellite_bundles [i]->culture, culture) == 0) { - char *bundle_name = g_strconcat (culture, "/", name, (const char *)NULL); + if (bundled_assembly_match (satellite_bundles[i]->name, filename) && strcmp (satellite_bundles [i]->culture, culture) == 0) { + // Since bundled images don't exist on disk, don't give them a legit filename image = mono_image_open_from_data_internal (alc, (char *)satellite_bundles [i]->data, satellite_bundles [i]->size, FALSE, status, FALSE, bundle_name, NULL); - g_free (bundle_name); - break; } } - g_free (name); + g_free (bundle_name); return image; } @@ -1530,12 +1524,16 @@ mono_assembly_open_from_bundle (MonoAssemblyLoadContext *alc, const char *filena MonoImage *image = NULL; MONO_ENTER_GC_UNSAFE; gboolean is_satellite = culture && culture [0] != 0; - - if (is_satellite) - image = open_from_satellite_bundle (alc, filename, status, culture); - else - image = open_from_bundle_internal (alc, filename, status, FALSE); - + const char *name; + if (is_satellite) { + name = g_strdup (filename); + image = open_from_satellite_bundle (alc, name, status, culture); + } + else { + name = g_path_get_basename (filename); + image = open_from_bundle_internal (alc, name, status); + } + g_free ((void *)name); if (image) { mono_image_addref (image); mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "Assembly Loader loaded assembly from bundle: '%s'.", filename); From ba241b5a044dc643492692112d6ff552b84059b6 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 21 Apr 2023 20:47:44 -0400 Subject: [PATCH 024/110] [mono] Incorporate new bundling APIs into bundled resource retrievals --- src/mono/mono/metadata/assembly-internals.h | 6 +++ src/mono/mono/metadata/assembly.c | 37 +++++++++++++++---- src/mono/mono/metadata/bundled-resources.c | 13 +++++++ src/mono/mono/metadata/mono-debug.c | 14 +++---- src/native/libs/System.Native/pal_datetime.c | 9 +++-- src/tasks/LibraryBuilder/Templates/autoinit.c | 11 +++--- .../Templates/library-builder.h | 4 -- 7 files changed, 65 insertions(+), 29 deletions(-) diff --git a/src/mono/mono/metadata/assembly-internals.h b/src/mono/mono/metadata/assembly-internals.h index f76a28fa283236..1d3e567b534d17 100644 --- a/src/mono/mono/metadata/assembly-internals.h +++ b/src/mono/mono/metadata/assembly-internals.h @@ -141,6 +141,12 @@ mono_assembly_get_image_internal (MonoAssembly *assembly); MonoBundledAssembly * mono_create_new_bundled_assembly (const char *name, const unsigned char *data, unsigned int size); +void +mono_hash_contains_bundled_assemblies (gboolean contains); + +void +mono_hash_contains_bundled_satellite_assemblies (gboolean contains); + void mono_set_assemblies_path_direct (char **path); diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 19645d35358f91..6e334fbac5e7e9 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -18,6 +18,7 @@ #include #include #include "assembly-internals.h" +#include "bundled-resources-internals.h" #include #include "image-internals.h" #include "object-internals.h" @@ -82,8 +83,9 @@ mono_assemblies_unlock (void) /* If defined, points to the bundled assembly information */ static const MonoBundledAssembly **bundles; - +static gboolean hash_contains_bundled_assemblies; static const MonoBundledSatelliteAssembly **satellite_bundles; +static gboolean hash_contains_bundled_satellite_assemblies; /* Class lazy loading functions */ static GENERATE_TRY_GET_CLASS_WITH_CACHE (debuggable_attribute, "System.Diagnostics", "DebuggableAttribute") @@ -710,7 +712,8 @@ mono_assembly_get_assemblyref (MonoImage *image, int index, MonoAssemblyName *an static MonoAssembly * search_bundle_for_assembly (MonoAssemblyLoadContext *alc, MonoAssemblyName *aname) { - if (bundles == NULL && satellite_bundles == NULL) + if (bundles == NULL && !hash_contains_bundled_assemblies && + satellite_bundles == NULL && !hash_contains_bundled_satellite_assemblies) return NULL; MonoImageOpenStatus status; @@ -793,7 +796,7 @@ netcore_load_reference (MonoAssemblyName *aname, MonoAssemblyLoadContext *alc, M } } - if (bundles != NULL && !is_satellite) { + if ((bundles != NULL || hash_contains_bundled_assemblies) && !is_satellite) { reference = search_bundle_for_assembly (mono_alc_get_default (), aname); if (reference) { mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "Assembly found in the bundle: '%s'.", aname->name); @@ -801,7 +804,7 @@ netcore_load_reference (MonoAssemblyName *aname, MonoAssemblyLoadContext *alc, M } } - if (satellite_bundles != NULL && is_satellite) { + if ((satellite_bundles != NULL || hash_contains_bundled_satellite_assemblies) && is_satellite) { // Satellite assembly byname requests should be loaded in the same ALC as their parent assembly size_t name_len = strlen (aname->name); char *parent_name = NULL; @@ -1472,9 +1475,13 @@ bundled_assembly_match (const char *bundled_name, const char *name) static MonoImage * open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status) { - if (!bundles) + if (!bundles && !hash_contains_bundled_assemblies) return NULL; + MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)mono_get_bundled_resource_data (filename); + if (assembly) + return mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, filename, NULL); + for (int i = 0; bundles [i]; ++i) { if (bundled_assembly_match (bundles[i]->name, filename)) { // Since bundled images don't exist on disk, don't give them a legit filename @@ -1488,12 +1495,16 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M static MonoImage * open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status, const char *culture) { - if (!satellite_bundles) + if (!satellite_bundles && !hash_contains_bundled_satellite_assemblies) return NULL; MonoImage *image = NULL; char *bundle_name = g_strconcat (culture, "/", filename, (const char *)NULL); + MonoBundledSatelliteAssemblyResource *satellite_assembly = (MonoBundledSatelliteAssemblyResource *)mono_get_bundled_resource_data (filename); + if (satellite_assembly) + image = mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, bundle_name, NULL); + for (int i = 0; !image && satellite_bundles [i]; ++i) { if (bundled_assembly_match (satellite_bundles[i]->name, filename) && strcmp (satellite_bundles [i]->culture, culture) == 0) { // Since bundled images don't exist on disk, don't give them a legit filename @@ -1610,7 +1621,7 @@ mono_assembly_request_open (const char *filename, const MonoAssemblyOpenRequest // If VM built with mkbundle loaded_from_bundle = FALSE; - if (bundles != NULL || satellite_bundles != NULL) { + if (bundles != NULL || hash_contains_bundled_assemblies) { /* We don't know the culture of the filename we're loading here, so this call is not culture aware. */ image = mono_assembly_open_from_bundle (load_req.alc, fname, status, NULL); loaded_from_bundle = image != NULL; @@ -3177,6 +3188,12 @@ mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) bundles = assemblies; } +void +mono_hash_contains_bundled_assemblies (gboolean contains) +{ + hash_contains_bundled_assemblies = contains; +} + /** * mono_create_new_bundled_satellite_assembly: */ @@ -3202,6 +3219,12 @@ mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly * satellite_bundles = assemblies; } +void +mono_hash_contains_bundled_satellite_assemblies (gboolean contains) +{ + hash_contains_bundled_satellite_assemblies = contains; +} + /** * mono_assembly_is_jit_optimizer_disabled: * diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 7064801b1fa8b4..60098307c9a498 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -24,6 +24,9 @@ mono_free_bundled_resources (void) { g_hash_table_destroy (bundled_resources); bundled_resources = NULL; + + mono_hash_contains_bundled_assemblies (FALSE); + mono_hash_contains_bundled_satellite_assemblies (FALSE); } //--------------------------------------------------------------------------------------- @@ -51,17 +54,21 @@ mono_add_bundled_resource (MonoBundledResource **resources_to_bundle, uint32_t l if (!bundled_resources) bundled_resources = g_hash_table_new (g_str_hash, g_str_equal); + gboolean assemblyAdded, satelliteAssemblyAdded; + for (int i = 0; i < len; ++i) { MonoBundledResource *resource_to_bundle = (MonoBundledResource *)resources_to_bundle[i]; switch (resource_to_bundle->type) { case MONO_BUNDLED_ASSEMBLY: { MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)resource_to_bundle; g_hash_table_insert (bundled_resources, (gpointer) assembly->assembly.name, assembly); + assemblyAdded = TRUE; break; } case MONO_BUNDLED_SATELLITE_ASSEMBLY: { MonoBundledSatelliteAssemblyResource *satellite_assembly = (MonoBundledSatelliteAssemblyResource *)resource_to_bundle; g_hash_table_insert (bundled_resources, (gpointer) satellite_assembly->satellite_assembly.name, satellite_assembly); + satelliteAssemblyAdded = TRUE; break; } case MONO_BUNDLED_DATA: @@ -72,6 +79,12 @@ mono_add_bundled_resource (MonoBundledResource **resources_to_bundle, uint32_t l } } } + + if (assemblyAdded) + mono_hash_contains_bundled_assemblies (assemblyAdded); + + if (satelliteAssemblyAdded) + mono_hash_contains_bundled_satellite_assemblies (satelliteAssemblyAdded); } //--------------------------------------------------------------------------------------- diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 8c9dfc5cf1ec0a..8bcd8caad03d2f 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1121,11 +1121,9 @@ open_symfile_from_bundle (MonoImage *image) { BundledSymfile *bsymfile; - const unsigned char *data; - unsigned int data_len; - mono_get_bundled_resource_data (image->module_name, &data, &data_len); - if (data) - return mono_debug_open_image (image, data, data_len); + MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)mono_get_bundled_resource_data (image->module_name); + if (assembly && assembly->symfile.data) + return mono_debug_open_image (image, assembly->symfile.data, assembly->symfile.size); #ifdef ENABLE_WEBCIL int len = strlen (image->module_name); @@ -1134,11 +1132,11 @@ open_symfile_from_bundle (MonoImage *image) if (module_name_dll_suffix && !g_strcasecmp (".webcil", &image->module_name [len - 7])) { memcpy (module_name_dll_suffix + len - 7, ".dll", 4); *(module_name_dll_suffix + len - 3) = '\0'; - mono_get_bundled_resource_data (module_name_dll_suffix, &data, &data_len); + assembly = (MonoBundledAssemblyResource *)mono_get_bundled_resource_data (module_name_dll_suffix); } g_free (module_name_dll_suffix); - if (data) - return mono_debug_open_image (image, data, data_len); + if (assembly && assembly->symfile) + return mono_debug_open_image (image, assembly->symfile.data, assembly->symfile.size); #endif for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { diff --git a/src/native/libs/System.Native/pal_datetime.c b/src/native/libs/System.Native/pal_datetime.c index 739e456ea7c709..81355a706e4cc7 100644 --- a/src/native/libs/System.Native/pal_datetime.c +++ b/src/native/libs/System.Native/pal_datetime.c @@ -21,7 +21,7 @@ static const int64_t TICKS_PER_MICROSECOND = 10; /* 1000 / 100 */ #endif #if defined(TARGET_WASI) || defined(TARGET_BROWSER) -extern void mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size) +extern MonoBundledResource * mono_get_bundled_resource_data (const char *name); #endif // @@ -67,9 +67,10 @@ const char* SystemNative_GetTimeZoneData(const char* name, int* length) assert(name != NULL); assert(length != NULL); #if defined(TARGET_WASI) || defined(TARGET_BROWSER) - const char *data; - mono_get_bundled_resource_data (name, &out_data, length) - return data; + MonoBundledDataResource *timezoneData = (MonoBundledDataResource *)mono_get_bundled_resource_data (name); + assert (timezoneData); + length = timezoneData->data.size; + return timezoneData->data.data; #else assert_msg(false, "Not supported on this platform", 0); (void)name; // unused diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index 74a4bee2bedf98..1fe544cbe26011 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -17,6 +17,7 @@ #include #include "library-builder.h" +#include "mono-bundled-source.h" static void cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) @@ -36,14 +37,12 @@ initialize_runtimeconfig (const char *bundle_path) LOG_ERROR ("Out of memory.\n"); const char *file_name = "runtimeconfig.bin"; - const unsigned char *data; - unsigned int data_len; - mono_get_bundled_resource_data (file_name, &data, &data_len); + MonoBundledDataResource *runtimeConfig = (MonoBundledDataResource *)mono_get_bundled_resource_data (file_name); - if (data) { + if (runtimeConfig) { arg->kind = 1; - arg->runtimeconfig.data.data = data; - arg->runtimeconfig.data.data_len = data_len; + arg->runtimeconfig.data.data = (const char *)runtimeConfig->data.data; + arg->runtimeconfig.data.data_len = runtimeConfig->data.size; } else { size_t str_len = sizeof (char) * (strlen (bundle_path) + strlen (file_name) + 2); // +1 "/", +1 null-terminating char char *file_path = (char *)malloc (str_len); diff --git a/src/tasks/LibraryBuilder/Templates/library-builder.h b/src/tasks/LibraryBuilder/Templates/library-builder.h index 5e7c2858542d7f..eae270c0048911 100644 --- a/src/tasks/LibraryBuilder/Templates/library-builder.h +++ b/src/tasks/LibraryBuilder/Templates/library-builder.h @@ -44,10 +44,6 @@ #endif // Platform specific native functions void register_aot_modules (void); -#if defined(BUNDLED_RESOURCES) -void mono_get_bundled_resource_data (const char *name, const unsigned char **out_data, unsigned int *out_size); -void mono_register_resources_bundle (void); -#endif // BUNDLED_RESOURCES void preload_assemblies_with_exported_symbols (); typedef void (*MonoRuntimeInitCallback) (void); void mono_set_runtime_init_callback (MonoRuntimeInitCallback callback); From cfa60e503756937b6fda49568d604814ec7ab6db Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 21 Apr 2023 17:01:48 -0400 Subject: [PATCH 025/110] [EmitBundle][task] Add templates for bundled source files Templates added: - header mirroring bundled-resources-internals.h and MonoBundledSatelliteAssembly - source file template for each resource's byte data and size - individual templates to build preallocated MonoBundled*Resource structs - source file template to hold preallocated MonoBundled*Resource structs --- src/tasks/EmitBundleTask/EmitBundle.csproj | 3 + .../Templates/mono-bundled-assembly.c | 6 ++ .../Templates/mono-bundled-data.c | 6 ++ .../mono-bundled-preallocated-source.c | 11 ++++ .../mono-bundled-satellite-assembly.c | 6 ++ .../Templates/mono-bundled-source.c | 3 + .../Templates/mono-bundled-source.h | 65 +++++++++++++++++++ .../Templates/mono-bundled-symfile.c | 2 + 8 files changed, 102 insertions(+) create mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.c create mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-data.c create mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c create mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.c create mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-source.c create mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-source.h create mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.c diff --git a/src/tasks/EmitBundleTask/EmitBundle.csproj b/src/tasks/EmitBundleTask/EmitBundle.csproj index 34b8fd649bf896..cfaa5eff214dc3 100644 --- a/src/tasks/EmitBundleTask/EmitBundle.csproj +++ b/src/tasks/EmitBundleTask/EmitBundle.csproj @@ -5,6 +5,9 @@ $(NoWarn),CA1050,CA1850 + + + diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.c new file mode 100644 index 00000000000000..941b9f93822325 --- /dev/null +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.c @@ -0,0 +1,6 @@ +const MonoBundledAssemblyResource %Symbol% = { + .resource = { .type = MONO_BUNDLED_ASSEMBLY }, + .assembly = { .name = "%RegisteredName%", + .data = %Symbol%_data, + .size = %Len% }, +%MonoBundledSymfile%}; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-data.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-data.c new file mode 100644 index 00000000000000..04cbe7b9ea03ac --- /dev/null +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-data.c @@ -0,0 +1,6 @@ +const MonoBundledDataResource %Symbol% = { + .resource = { .type = MONO_BUNDLED_DATA }, + .data = { .name = "%RegisteredName%", + .data = %Symbol%_data, + .size = %Len% }, +}; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c new file mode 100644 index 00000000000000..841902d5e94153 --- /dev/null +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#ifndef __MONO_BUNDLED_PREALLOCATED_SOURCE_H__ +#define __MONO_BUNDLED_PREALLOCATED_SOURCE_H__ + +#include "mono-bundled-source.h" + +%PreallocatedStructs% + +#endif /* __MONO_BUNDLED_PREALLOCATED_SOURCE_H__ */ \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.c new file mode 100644 index 00000000000000..deb208bc2fa9d9 --- /dev/null +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.c @@ -0,0 +1,6 @@ +const MonoBundledSatelliteAssemblyResource %Symbol% = { + .resource = { .type = MONO_BUNDLED_SATELLITE_ASSEMBLY }, + .satellite_assembly = { .name = "%RegisteredName%", + .data = %Symbol%_data, + .size = %Len% }, +%MonoBundledSymfile%}; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.c new file mode 100644 index 00000000000000..f233f035347251 --- /dev/null +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.c @@ -0,0 +1,3 @@ +unsigned char %Symbol%_data[] = {%Data% 0 +}; +unsigned int %Symbol%_len = %Length%; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h new file mode 100644 index 00000000000000..3a73328e1ae6ca --- /dev/null +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h @@ -0,0 +1,65 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#ifndef __MONO_BUNDLED_SOURCE_H__ +#define __MONO_BUNDLED_SOURCE_H__ + +#include + +typedef enum { + MONO_BUNDLED_DATA, + MONO_BUNDLED_ASSEMBLY, + MONO_BUNDLED_SATELLITE_ASSEMBLY, + MONO_BUNDLED_RESOURCE_COUNT, +} MonoBundledResourceType; + +typedef struct _MonoBundledResource { + MonoBundledResourceType type; + void (*free_bundled_resource_func)(void *); +} MonoBundledResource; + +typedef struct _MonoBundledData { + char *name; + const unsigned char *data; + unsigned int size; +} MonoBundledData; + +typedef struct _MonoBundledDataResource { + MonoBundledResource resource; + MonoBundledData data; +} MonoBundledDataResource; + +typedef struct _MonoBundledSymfile { + const unsigned char *data; + unsigned int size; +} MonoBundledSymfile; + +typedef struct _MonoBundledAssemblyResource { + MonoBundledResource resource; + MonoBundledAssembly assembly; + MonoBundledSymfile symfile; +} MonoBundledAssemblyResource; + +typedef struct _MonoBundledSatelliteAssembly { + const char *name; + const char *culture; + const unsigned char *data; + unsigned int size; +} MonoBundledSatelliteAssembly; + +typedef struct _MonoBundledSatelliteAssemblyResource { + MonoBundledResource resource; + MonoBundledSatelliteAssembly satellite_assembly; + MonoBundledSymfile symfile; +} MonoBundledSatelliteAssemblyResource; + +void +mono_free_bundled_resources (void); + +void +mono_add_bundled_resource (MonoBundledResource **resources_to_bundle, uint32_t len); + +MonoBundledResource * +mono_get_bundled_resource_data (const char *name); + +#endif /* __MONO_BUNDLED_SOURCE_H__ */ \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.c new file mode 100644 index 00000000000000..640b74ea138b62 --- /dev/null +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.c @@ -0,0 +1,2 @@ + .symfile = { .data = &%SymfileSymbol%_data, + .size = %SymLen% } From 6fe991259dae1815d1da92caafa1e00250075655 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 21 Apr 2023 18:39:17 -0400 Subject: [PATCH 026/110] [EmitBundle] Enable consolidation of bundled resource symbols --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 22 ++++++++++++++++--- .../EmitBundleTask/EmitBundleSourceFiles.cs | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 5e8bebc74e0f0c..0cf69f9bb30c27 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -27,6 +27,18 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab [Required] public string BundleFile { get; set; } = default!; + /// + /// Filename for the unified source containing all byte data and len + /// of the FilesToBundle resources. + /// Leave empty to keep sources separate. + /// + public string? CombinedResourceSource { get; set; } + + /// + /// Path to store build artifacts + /// + public string OutputDirectory {get; set; } = default!; + public override bool Execute() { // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore @@ -68,6 +80,8 @@ public override bool Execute() int allowedParallelism = Math.Max(Math.Min(remainingDestinationFilesToBundle.Length, Environment.ProcessorCount), 1); if (BuildEngine is IBuildEngine9 be9) allowedParallelism = be9.RequestCores(allowedParallelism); + if (!string.IsNullOrEmpty(CombinedResourceSource)) + allowedParallelism = 1; Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => { @@ -78,6 +92,8 @@ public override bool Execute() var contentSourceFile = group.First(); var outputFile = group.Key; + if (!string.IsNullOrEmpty(CombinedResourceSource)) + outputFile = Path.Combine(OutputDirectory, CombinedResourceSource); var inputFile = contentSourceFile.ItemSpec; if (verbose) { @@ -90,8 +106,8 @@ public override bool Execute() Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); } - Log.LogMessage(MessageImportance.Low, "Bundling {0} as {1}", inputFile, outputFile); - var symbolName = ToSafeSymbolName(outputFile); + Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, outputFile); + var symbolName = ToSafeSymbolName(group.Key); if (!Emit(outputFile, (codeStream) => { using var inputStream = File.OpenRead(inputFile); BundleFileToCSource(symbolName, inputStream, codeStream); @@ -188,7 +204,7 @@ private static void BundleFileToCSource(string symbolName, FileStream inputStrea using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); - outputUtf8Writer.Write($"unsigned char {symbolName}[] = {{"); + outputUtf8Writer.Write($"unsigned char {symbolName}_data[] = {{"); outputUtf8Writer.Flush(); while ((bytesRead = inputStream.Read(buf, 0, buf.Length)) > 0) { diff --git a/src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs b/src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs index 5e256c4eb66354..3ff510502222d1 100644 --- a/src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs +++ b/src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs @@ -12,7 +12,7 @@ public class EmitBundleSourceFiles : EmitBundleBase { public override bool Emit(string destinationFile, Action inputProvider) { - using (var fileStream = File.Create(destinationFile)) + using (var fileStream = new FileStream(destinationFile, FileMode.Append, FileAccess.Write)) { inputProvider(fileStream); } From 077c020c41d69d997b3d4bbf4a5c5191936acd61 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 21 Apr 2023 18:59:09 -0400 Subject: [PATCH 027/110] [EmitBundle] Generate bundled source file to preallocate MonoBundled*Resources --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 91 ++++++++++++++-------- 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 0cf69f9bb30c27..ddaf0a026eccca 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -21,12 +21,6 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab [Required] public ITaskItem[] FilesToBundle { get; set; } = default!; - [Required] - public string BundleName { get; set; } = default!; - - [Required] - public string BundleFile { get; set; } = default!; - /// /// Filename for the unified source containing all byte data and len /// of the FilesToBundle resources. @@ -69,12 +63,13 @@ public override bool Execute() var outputFile = registeredFile.GetMetadata("DestinationFile"); var registeredName = group.Key; var symbolName = ToSafeSymbolName(outputFile); - var fileType = GetFileType(registeredFile.ItemSpec); - return (registeredName, symbolName, fileType); + string? symfileSymbolName = null; + if (File.Exists(registeredFile.GetMetadata("Symfile"))) + symfileSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("Symfile")); + return (registeredName, symbolName, symfileSymbolName); }).ToList(); - Log.LogMessage(MessageImportance.Low, "Bundling {0} files for {1}", files.Count, BundleName); - + // Generate source file(s) containing each resource's byte data and size if (remainingDestinationFilesToBundle.Length > 0) { int allowedParallelism = Math.Max(Math.Min(remainingDestinationFilesToBundle.Length, Environment.ProcessorCount), 1); @@ -118,11 +113,13 @@ public override bool Execute() }); } - return Emit(BundleFile, (inputStream) => - { - using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); - GenerateRegisteredBundledObjects($"mono_register_{BundleName}_bundle", files, outputUtf8Writer); - }) && !Log.HasLoggedErrors; + // Generate header containing MonoBundled*Resource typedefs + File.WriteAllText(Path.Combine(OutputDirectory, "mono-bundled-source.h"), Utils.GetEmbeddedResource("mono-bundled-source.h")); + + // Generate source file containing preallocated MonoBundled*Resource structs + GeneratePreallocatedMonoBundleStructs(OutputDirectory, files); + + return true; } public void Cancel() @@ -162,30 +159,56 @@ private static byte[] InitLookupTable() public abstract bool Emit(string destinationFile, Action inputProvider); - public static void GenerateRegisteredBundledObjects(string newFunctionName, ICollection<(string registeredName, string symbol, string type)> files, StreamWriter outputUtf8Writer) - { - outputUtf8Writer.WriteLine("typedef enum {\n MONO_BUNDLED_DATA,\n MONO_BUNDLED_ASSEMBLY,\n MONO_BUNDLED_SATELLITE_ASSEMBLY,\n MONO_BUNDLED_PDB,\n MONO_BUNDLED_RESOURCE_COUNT,\n} MonoBundledResourceType;"); - outputUtf8Writer.WriteLine($"void mono_add_bundled_resource(const char *name, const char *culture, const unsigned char *data, unsigned int size, MonoBundledResourceType type);"); - outputUtf8Writer.WriteLine($"void mono_register_bundled_resources (void);"); - outputUtf8Writer.WriteLine(); - - foreach (var tuple in files) - { - outputUtf8Writer.WriteLine($"extern const unsigned char {tuple.symbol}[];"); - outputUtf8Writer.WriteLine($"extern const int {tuple.symbol}_len;"); - } + private static Dictionary symbolDataLen = new(); - outputUtf8Writer.WriteLine(); - outputUtf8Writer.WriteLine($"void {newFunctionName}() {{"); + private static void GeneratePreallocatedMonoBundleStructs(string outputDir, ICollection<(string registeredName, string symbol, string? symfileSymbol)> files) + { + StringBuilder preallocatedSource = new(); foreach (var tuple in files) { - outputUtf8Writer.WriteLine($" mono_add_bundled_resource (\"{tuple.registeredName}\", 0, {tuple.symbol}, {tuple.symbol}_len, {tuple.type});"); + // extern symbols + StringBuilder preallocatedData = new(); + preallocatedData.AppendLine($"extern const unsigned char {tuple.symbol}_data[];"); + if (!string.IsNullOrEmpty(tuple.symfileSymbol)) + { + preallocatedData.AppendLine($"extern const unsigned char {tuple.symfileSymbol}_data[];"); + } + preallocatedSource.AppendLine(preallocatedData.ToString()); + + // Generate Preloaded MonoBundled*Resource structs + string preloadedStruct; + switch (GetFileType(tuple.registeredName)) { + case "MONO_BUNDLED_ASSEMBLY": { + preloadedStruct = Utils.GetEmbeddedResource("mono-bundled-assembly.c"); + break; + } + case "MONO_BUNDLED_SATELLITE_ASSEMBLY": { + preloadedStruct = Utils.GetEmbeddedResource("mono-bundled-satellite-assembly.c"); + break; + } + case "MONO_BUNDLED_DATA": + default: { + preloadedStruct = Utils.GetEmbeddedResource("mono-bundled-data.c"); + break; + } + } + // Add associated symfile information to MonoBundledAssemblyResource/MonoBundleSatelliteAssemblyResource structs + string preloadedSymfile = ""; + if (!string.IsNullOrEmpty(tuple.symfileSymbol)) + { + preloadedSymfile = Utils.GetEmbeddedResource("mono-bundled-symfile.c") + .Replace("%SymfileSymbol%", tuple.symfileSymbol) + .Replace("%SymLen%", symbolDataLen[tuple.symfileSymbol].ToString()); + } + preallocatedSource.AppendLine(preloadedStruct.Replace("%RegisteredName%", tuple.registeredName) + .Replace("%Symbol%", tuple.symbol) + .Replace("%Len%", symbolDataLen[tuple.symbol].ToString()) + .Replace("%MonoBundledSymfile%", preloadedSymfile)); } - outputUtf8Writer.WriteLine($" mono_register_bundled_resources ();"); - - outputUtf8Writer.WriteLine("}"); + File.WriteAllText(Path.Combine(outputDir, "mono-bundled-preallocated-source.c"), Utils.GetEmbeddedResource("mono-bundled-preallocated-source.c") + .Replace("%PreallocatedStructs%", preallocatedSource.ToString())); } private static void BundleFileToCSource(string symbolName, FileStream inputStream, Stream outputStream) @@ -226,6 +249,8 @@ private static void BundleFileToCSource(string symbolName, FileStream inputStrea outputUtf8Writer.WriteLine($"unsigned int {symbolName}_len = {generatedArrayLength};"); outputUtf8Writer.Flush(); outputStream.Flush(); + + symbolDataLen.Add(symbolName, generatedArrayLength); } private static string ToSafeSymbolName(string destinationFileName) From 1e7ad1c230e026ebfc8b3d31c4c35a03e084b301 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 21 Apr 2023 19:06:33 -0400 Subject: [PATCH 028/110] [EmitBundle] Output symbol to resource type map for populating function to add bundled resources --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index ddaf0a026eccca..f93120bafac88a 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -10,6 +11,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelableTask { @@ -33,6 +35,9 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab /// public string OutputDirectory {get; set; } = default!; + [Output] + public ITaskItem[] PreallocatedSymbols { get; set; } = Array.Empty(); + public override bool Execute() { // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore @@ -117,7 +122,7 @@ public override bool Execute() File.WriteAllText(Path.Combine(OutputDirectory, "mono-bundled-source.h"), Utils.GetEmbeddedResource("mono-bundled-source.h")); // Generate source file containing preallocated MonoBundled*Resource structs - GeneratePreallocatedMonoBundleStructs(OutputDirectory, files); + PreallocatedSymbols = GeneratePreallocatedMonoBundleStructs(OutputDirectory, files); return true; } @@ -161,12 +166,15 @@ private static byte[] InitLookupTable() private static Dictionary symbolDataLen = new(); - private static void GeneratePreallocatedMonoBundleStructs(string outputDir, ICollection<(string registeredName, string symbol, string? symfileSymbol)> files) + private static ITaskItem[] GeneratePreallocatedMonoBundleStructs(string outputDir, ICollection<(string registeredName, string symbol, string? symfileSymbol)> files) { + var preallocatedSymbols = new ArrayList(); StringBuilder preallocatedSource = new(); foreach (var tuple in files) { + var preallocatedSymbol = new TaskItem(tuple.symbol); + // extern symbols StringBuilder preallocatedData = new(); preallocatedData.AppendLine($"extern const unsigned char {tuple.symbol}_data[];"); @@ -181,15 +189,18 @@ private static void GeneratePreallocatedMonoBundleStructs(string outputDir, ICol switch (GetFileType(tuple.registeredName)) { case "MONO_BUNDLED_ASSEMBLY": { preloadedStruct = Utils.GetEmbeddedResource("mono-bundled-assembly.c"); + preallocatedSymbol.SetMetadata("ResourceType", "MonoBundledAssemblyResource"); break; } case "MONO_BUNDLED_SATELLITE_ASSEMBLY": { preloadedStruct = Utils.GetEmbeddedResource("mono-bundled-satellite-assembly.c"); + preallocatedSymbol.SetMetadata("ResourceType", "MonoBundledSatelliteAssemblyResource"); break; } case "MONO_BUNDLED_DATA": default: { preloadedStruct = Utils.GetEmbeddedResource("mono-bundled-data.c"); + preallocatedSymbol.SetMetadata("ResourceType", "MonoBundledDataResource"); break; } } @@ -205,10 +216,14 @@ private static void GeneratePreallocatedMonoBundleStructs(string outputDir, ICol .Replace("%Symbol%", tuple.symbol) .Replace("%Len%", symbolDataLen[tuple.symbol].ToString()) .Replace("%MonoBundledSymfile%", preloadedSymfile)); + + preallocatedSymbols.Add(preallocatedSymbol); } File.WriteAllText(Path.Combine(outputDir, "mono-bundled-preallocated-source.c"), Utils.GetEmbeddedResource("mono-bundled-preallocated-source.c") .Replace("%PreallocatedStructs%", preallocatedSource.ToString())); + + return (ITaskItem[])preallocatedSymbols.ToArray(typeof(ITaskItem)); } private static void BundleFileToCSource(string symbolName, FileStream inputStream, Stream outputStream) From c5eb985ac48c5b5650d4820215df66dfcf18e1a3 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 21 Apr 2023 21:00:16 -0400 Subject: [PATCH 029/110] [LibraryBuilder] Generate source file to add preallocated MonoBundled*Resources --- src/tasks/LibraryBuilder/LibraryBuilder.cs | 67 +++++++++++++++++++ .../Templates/preallocated-resources.c | 11 +++ 2 files changed, 78 insertions(+) create mode 100644 src/tasks/LibraryBuilder/Templates/preallocated-resources.c diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index 8419718b37fe3d..93fd60c379651e 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -90,6 +90,11 @@ public bool IsSharedLibrary /// public bool BundlesResources { get; set; } + /// + /// The symbols of preallocated structs being bundled + /// + public ITaskItem[] BundledPreallocatedSymbols { get; set; } = Array.Empty(); + public bool StripDebugSymbols { get; set; } /// @@ -154,6 +159,12 @@ public override bool Execute() extraSources.AppendLine(" autoinit.c"); } + if (BundlesResources) + { + GenerateBundledResourcesLoader(); + extraSources.AppendLine(" preallocated-resources.c"); + } + WriteCMakeFileFromTemplate(aotSources.ToString(), aotObjects.ToString(), extraSources.ToString(), linkerArgs.ToString()); OutputPath = BuildLibrary(); @@ -297,6 +308,62 @@ private void GenerateAssembliesLoader() .Replace("%ASSEMBLIES_PRELOADER%", string.Join("\n ", assemblyPreloaders))); } + private void GenerateBundledResourcesLoader() + { + var preallocatedResources = new StringBuilder(); + var preallocatedAssemblies = new StringBuilder("MonoBundledResource *bundledAssemblyResources[] = { "); + var preallocatedSatelliteAssemblies = new StringBuilder("MonoBundledResource *bundledSatelliteAssemblyResources[] = { "); + var preallocatedData = new StringBuilder("MonoBundledResource *bundledDataResources[] = { "); + int assembliesCount = 0; + int satelliteAssembliesCount = 0; + int dataCount = 0; + foreach (ITaskItem bundledPreallocatedSymbol in BundledPreallocatedSymbols) + { + preallocatedResources.AppendLine($"extern const {bundledPreallocatedSymbol.GetMetadata("ResourceType")} {bundledPreallocatedSymbol.ItemSpec};"); + + switch (bundledPreallocatedSymbol.GetMetadata("ResourceType")) { + case "MonoBundledAssemblyResource": { + preallocatedAssemblies.Append($"(MonoBundledResource *)&{bundledPreallocatedSymbol.ItemSpec}, "); + assembliesCount += 1; + break; + } + case "MonoBundledSatelliteAssemblyResource": { + preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{bundledPreallocatedSymbol.ItemSpec}, "); + satelliteAssembliesCount += 1; + break; + } + case "MonoBundledDataResource": + default: { + preallocatedData.Append($"(MonoBundledResource *)&{bundledPreallocatedSymbol.ItemSpec}, "); + dataCount += 1; + break; + } + } + } + + var addPreallocatedResources = new StringBuilder(); + if (assembliesCount != 0) { + preallocatedAssemblies.AppendLine("};"); + preallocatedResources.AppendLine(preallocatedAssemblies.ToString()); + addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledAssemblyResources, {assembliesCount});"); + } + if (satelliteAssembliesCount != 0) { + preallocatedSatelliteAssemblies.AppendLine("};"); + preallocatedResources.AppendLine(preallocatedSatelliteAssemblies.ToString()); + addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledSatelliteAssemblyResources, {satelliteAssembliesCount});"); + } + if (dataCount != 0) { + preallocatedData.AppendLine("};"); + preallocatedResources.AppendLine(preallocatedData.ToString()); + addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledDataResources, {dataCount});"); + } + + File.WriteAllText(Path.Combine(OutputDirectory, "preallocated-resources.c"), + Utils.GetEmbeddedResource("preallocated-resources.c") + .Replace("%PreallocatedResources%", preallocatedResources.ToString()) + .Replace("%AddPreallocatedResources%", addPreallocatedResources.ToString())); + } + private void WriteCMakeFileFromTemplate(string aotSources, string aotObjects, string extraSources, string linkerArgs) { string extraDefinitions = GenerateExtraDefinitions(); diff --git a/src/tasks/LibraryBuilder/Templates/preallocated-resources.c b/src/tasks/LibraryBuilder/Templates/preallocated-resources.c new file mode 100644 index 00000000000000..7e05eba975397b --- /dev/null +++ b/src/tasks/LibraryBuilder/Templates/preallocated-resources.c @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "mono-bundled-source.h" + +%PreallocatedResources% + +void +mono_register_resources_bundle (void) { +%AddPreallocatedResources% +} From d28ee4be45ca3cb28bced46819bd4c43a52d6431 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 21 Apr 2023 20:51:34 -0400 Subject: [PATCH 030/110] [LibraryBuilder] Incorporate EmitBundle templates into targets flow --- .../msbuild/common/LibraryBuilder.targets | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index dfb9d482af74bd..f09083dd20e49a 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -19,8 +19,7 @@ <_ExtraLibrarySources Include="$(_AotModuleTablePath)" /> - <_ExtraLibrarySources Include="$(_ResourcesToBundleSourceFile)" /> - <_ExtraLibrarySources Include="%(_ResourcesToBundleWithHashes.DestinationFile)" /> + <_ExtraLibrarySources Include="@(BundledSources)" /> <_ExtraLinkerArgs Include="@(_CommonLinkerArgs)" /> @@ -28,6 +27,7 @@ Assemblies="@(_AssembliesToBundleInternal)" AssembliesLocation="$(AssembliesLocation)" BundlesResources="$(_BundlesResources)" + BundledPreallocatedSymbols="@(BundledPreallocatedSymbols)" ExtraLinkerArguments="@(_ExtraLinkerArgs)" ExtraSources="@(_ExtraLibrarySources)" IsSharedLibrary="$(_IsSharedLibrary)" @@ -46,11 +46,17 @@ - - <_ResourcesToBundleSourceFile>$(BundleDir)bundled_resources.c + mono-bundled-source.c + $(BundleDir) + + + + + + <_ResourcesToBundle Remove="@(_ResourcesToBundle)" /> <_ResourcesToBundle Include="@(_AssembliesToBundleInternal)" /> @@ -63,15 +69,18 @@ <_ResourcesToBundleWithHashes Update="@(_ResourcesToBundleWithHashes)"> - $(BundleDir)%(_ResourcesToBundleWithHashes.Filename)%(_ResourcesToBundleWithHashes.Extension).$([System.String]::Copy(%(_ResourcesToBundleWithHashes.FileHash)).Substring(0, 8)).c + $(BundleOutputDirectory)%(_ResourcesToBundleWithHashes.Filename)%(_ResourcesToBundleWithHashes.Extension).$([System.String]::Copy(%(_ResourcesToBundleWithHashes.FileHash)).Substring(0, 8)).c + + + CombinedResourceSource="$(BundledSourceFile)" + OutputDirectory="$(BundleOutputDirectory)"> + + \ No newline at end of file From a796bfbd69279bdbd5a19936e99a24333a9348ca Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Mon, 24 Apr 2023 15:24:57 -0400 Subject: [PATCH 031/110] [EmitBundle] Move bundle registration back into EmitBundle [EmitBundle] Leverage Emit for file generation --- .../msbuild/common/LibraryBuilder.targets | 1 - src/tasks/EmitBundleTask/EmitBundleBase.cs | 105 ++++++++++++++++-- .../mono-bundled-preallocated-source.c | 7 +- .../mono-bundled-resource-registration.c} | 2 +- src/tasks/LibraryBuilder/LibraryBuilder.cs | 67 ----------- 5 files changed, 99 insertions(+), 83 deletions(-) rename src/tasks/{LibraryBuilder/Templates/preallocated-resources.c => EmitBundleTask/Templates/mono-bundled-resource-registration.c} (84%) diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index f09083dd20e49a..a7db86579dd9ae 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -27,7 +27,6 @@ Assemblies="@(_AssembliesToBundleInternal)" AssembliesLocation="$(AssembliesLocation)" BundlesResources="$(_BundlesResources)" - BundledPreallocatedSymbols="@(BundledPreallocatedSymbols)" ExtraLinkerArguments="@(_ExtraLinkerArgs)" ExtraSources="@(_ExtraLibrarySources)" IsSharedLibrary="$(_IsSharedLibrary)" diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index f93120bafac88a..23204da3b44c16 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -16,6 +16,7 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelableTask { private CancellationTokenSource BuildTaskCancelled { get; } = new(); + private static ITaskItem[] PreallocatedSymbols { get; set; } = Array.Empty(); /// Must have DestinationFile metadata, which is the output filename /// Could have RegisteredName, otherwise it would be the filename. @@ -23,6 +24,30 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab [Required] public ITaskItem[] FilesToBundle { get; set; } = default!; + /// + /// The function to call before mono runtime initialization + /// in order to register the bundled resources in FilesToBundle + /// + public string BundleRegistrationFunctionName { get; set; } = "mono_register_resources_bundle"; + + /// + /// The filename for the generated source file that registers + /// the bundled resources. + /// + public string BundleFile { get; set; } = "mono-bundled-resource-registration.c"; + + /// + /// The filename for the generated header file that declares + /// MonoBundled*Resource struct types. + /// + public string BundleHeader { get; set; } = "mono-bundled-source.h"; + + /// + /// The filename for the generated source file that preallocates + /// MonoBundled*Resource structs. + /// + public string BundlePreallocationFile { get; set; } = "mono-bundled-preallocated-source.c"; + /// /// Filename for the unified source containing all byte data and len /// of the FilesToBundle resources. @@ -35,9 +60,6 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab /// public string OutputDirectory {get; set; } = default!; - [Output] - public ITaskItem[] PreallocatedSymbols { get; set; } = Array.Empty(); - public override bool Execute() { // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore @@ -119,10 +141,21 @@ public override bool Execute() } // Generate header containing MonoBundled*Resource typedefs - File.WriteAllText(Path.Combine(OutputDirectory, "mono-bundled-source.h"), Utils.GetEmbeddedResource("mono-bundled-source.h")); + File.WriteAllText(Path.Combine(OutputDirectory, BundleHeader), Utils.GetEmbeddedResource("mono-bundled-source.h")); // Generate source file containing preallocated MonoBundled*Resource structs - PreallocatedSymbols = GeneratePreallocatedMonoBundleStructs(OutputDirectory, files); + Emit(Path.Combine(OutputDirectory, BundlePreallocationFile), (inputStream) => + { + using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); + GeneratePreallocatedMonoBundleStructs(outputUtf8Writer, files); + }); + + // Generate source file to register bundled resources + Emit(Path.Combine(OutputDirectory, BundleFile), (inputStream) => + { + using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); + GenerateBundledResourcesRegistration(BundleRegistrationFunctionName, outputUtf8Writer); + }); return true; } @@ -166,7 +199,7 @@ private static byte[] InitLookupTable() private static Dictionary symbolDataLen = new(); - private static ITaskItem[] GeneratePreallocatedMonoBundleStructs(string outputDir, ICollection<(string registeredName, string symbol, string? symfileSymbol)> files) + private static void GeneratePreallocatedMonoBundleStructs(StreamWriter outputUtf8Writer, ICollection<(string registeredName, string symbol, string? symfileSymbol)> files) { var preallocatedSymbols = new ArrayList(); StringBuilder preallocatedSource = new(); @@ -220,10 +253,66 @@ private static ITaskItem[] GeneratePreallocatedMonoBundleStructs(string outputDi preallocatedSymbols.Add(preallocatedSymbol); } - File.WriteAllText(Path.Combine(outputDir, "mono-bundled-preallocated-source.c"), Utils.GetEmbeddedResource("mono-bundled-preallocated-source.c") + outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-preallocated-source.c") .Replace("%PreallocatedStructs%", preallocatedSource.ToString())); - return (ITaskItem[])preallocatedSymbols.ToArray(typeof(ITaskItem)); + PreallocatedSymbols = (ITaskItem[])preallocatedSymbols.ToArray(typeof(ITaskItem)); + } + + private static void GenerateBundledResourcesRegistration(string bundleRegistrationFunctionName, StreamWriter outputUtf8Writer) + { + var preallocatedResources = new StringBuilder(); + var preallocatedAssemblies = new StringBuilder("MonoBundledResource *bundledAssemblyResources[] = { "); + var preallocatedSatelliteAssemblies = new StringBuilder("MonoBundledResource *bundledSatelliteAssemblyResources[] = { "); + var preallocatedData = new StringBuilder("MonoBundledResource *bundledDataResources[] = { "); + int assembliesCount = 0; + int satelliteAssembliesCount = 0; + int dataCount = 0; + foreach (ITaskItem preallocatedSymbolItem in PreallocatedSymbols) + { + preallocatedResources.AppendLine($"extern const {preallocatedSymbolItem.GetMetadata("ResourceType")} {preallocatedSymbolItem.ItemSpec};"); + + switch (preallocatedSymbolItem.GetMetadata("ResourceType")) { + case "MonoBundledAssemblyResource": { + preallocatedAssemblies.Append($"(MonoBundledResource *)&{preallocatedSymbolItem.ItemSpec}, "); + assembliesCount += 1; + break; + } + case "MonoBundledSatelliteAssemblyResource": { + preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{preallocatedSymbolItem.ItemSpec}, "); + satelliteAssembliesCount += 1; + break; + } + case "MonoBundledDataResource": + default: { + preallocatedData.Append($"(MonoBundledResource *)&{preallocatedSymbolItem.ItemSpec}, "); + dataCount += 1; + break; + } + } + } + + var addPreallocatedResources = new StringBuilder(); + if (assembliesCount != 0) { + preallocatedAssemblies.AppendLine("};"); + preallocatedResources.AppendLine(preallocatedAssemblies.ToString()); + addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledAssemblyResources, {assembliesCount});"); + } + if (satelliteAssembliesCount != 0) { + preallocatedSatelliteAssemblies.AppendLine("};"); + preallocatedResources.AppendLine(preallocatedSatelliteAssemblies.ToString()); + addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledSatelliteAssemblyResources, {satelliteAssembliesCount});"); + } + if (dataCount != 0) { + preallocatedData.AppendLine("};"); + preallocatedResources.AppendLine(preallocatedData.ToString()); + addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledDataResources, {dataCount});"); + } + + outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-resource-registration.c") + .Replace("%PreallocatedResources%", preallocatedResources.ToString()) + .Replace("%BundleRegistrationFunctionName%", bundleRegistrationFunctionName) + .Replace("%AddPreallocatedResources%", addPreallocatedResources.ToString())); } private static void BundleFileToCSource(string symbolName, FileStream inputStream, Stream outputStream) diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c index 841902d5e94153..b312e5c1084196 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c @@ -1,11 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#ifndef __MONO_BUNDLED_PREALLOCATED_SOURCE_H__ -#define __MONO_BUNDLED_PREALLOCATED_SOURCE_H__ - #include "mono-bundled-source.h" -%PreallocatedStructs% - -#endif /* __MONO_BUNDLED_PREALLOCATED_SOURCE_H__ */ \ No newline at end of file +%PreallocatedStructs% \ No newline at end of file diff --git a/src/tasks/LibraryBuilder/Templates/preallocated-resources.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-registration.c similarity index 84% rename from src/tasks/LibraryBuilder/Templates/preallocated-resources.c rename to src/tasks/EmitBundleTask/Templates/mono-bundled-resource-registration.c index 7e05eba975397b..b9ba67f58761d1 100644 --- a/src/tasks/LibraryBuilder/Templates/preallocated-resources.c +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-registration.c @@ -6,6 +6,6 @@ %PreallocatedResources% void -mono_register_resources_bundle (void) { +%BundleRegistrationFunctionName% (void) { %AddPreallocatedResources% } diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index 93fd60c379651e..8419718b37fe3d 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -90,11 +90,6 @@ public bool IsSharedLibrary /// public bool BundlesResources { get; set; } - /// - /// The symbols of preallocated structs being bundled - /// - public ITaskItem[] BundledPreallocatedSymbols { get; set; } = Array.Empty(); - public bool StripDebugSymbols { get; set; } /// @@ -159,12 +154,6 @@ public override bool Execute() extraSources.AppendLine(" autoinit.c"); } - if (BundlesResources) - { - GenerateBundledResourcesLoader(); - extraSources.AppendLine(" preallocated-resources.c"); - } - WriteCMakeFileFromTemplate(aotSources.ToString(), aotObjects.ToString(), extraSources.ToString(), linkerArgs.ToString()); OutputPath = BuildLibrary(); @@ -308,62 +297,6 @@ private void GenerateAssembliesLoader() .Replace("%ASSEMBLIES_PRELOADER%", string.Join("\n ", assemblyPreloaders))); } - private void GenerateBundledResourcesLoader() - { - var preallocatedResources = new StringBuilder(); - var preallocatedAssemblies = new StringBuilder("MonoBundledResource *bundledAssemblyResources[] = { "); - var preallocatedSatelliteAssemblies = new StringBuilder("MonoBundledResource *bundledSatelliteAssemblyResources[] = { "); - var preallocatedData = new StringBuilder("MonoBundledResource *bundledDataResources[] = { "); - int assembliesCount = 0; - int satelliteAssembliesCount = 0; - int dataCount = 0; - foreach (ITaskItem bundledPreallocatedSymbol in BundledPreallocatedSymbols) - { - preallocatedResources.AppendLine($"extern const {bundledPreallocatedSymbol.GetMetadata("ResourceType")} {bundledPreallocatedSymbol.ItemSpec};"); - - switch (bundledPreallocatedSymbol.GetMetadata("ResourceType")) { - case "MonoBundledAssemblyResource": { - preallocatedAssemblies.Append($"(MonoBundledResource *)&{bundledPreallocatedSymbol.ItemSpec}, "); - assembliesCount += 1; - break; - } - case "MonoBundledSatelliteAssemblyResource": { - preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{bundledPreallocatedSymbol.ItemSpec}, "); - satelliteAssembliesCount += 1; - break; - } - case "MonoBundledDataResource": - default: { - preallocatedData.Append($"(MonoBundledResource *)&{bundledPreallocatedSymbol.ItemSpec}, "); - dataCount += 1; - break; - } - } - } - - var addPreallocatedResources = new StringBuilder(); - if (assembliesCount != 0) { - preallocatedAssemblies.AppendLine("};"); - preallocatedResources.AppendLine(preallocatedAssemblies.ToString()); - addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledAssemblyResources, {assembliesCount});"); - } - if (satelliteAssembliesCount != 0) { - preallocatedSatelliteAssemblies.AppendLine("};"); - preallocatedResources.AppendLine(preallocatedSatelliteAssemblies.ToString()); - addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledSatelliteAssemblyResources, {satelliteAssembliesCount});"); - } - if (dataCount != 0) { - preallocatedData.AppendLine("};"); - preallocatedResources.AppendLine(preallocatedData.ToString()); - addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledDataResources, {dataCount});"); - } - - File.WriteAllText(Path.Combine(OutputDirectory, "preallocated-resources.c"), - Utils.GetEmbeddedResource("preallocated-resources.c") - .Replace("%PreallocatedResources%", preallocatedResources.ToString()) - .Replace("%AddPreallocatedResources%", addPreallocatedResources.ToString())); - } - private void WriteCMakeFileFromTemplate(string aotSources, string aotObjects, string extraSources, string linkerArgs) { string extraDefinitions = GenerateExtraDefinitions(); From 364a119ba67c482a58a823b77089ecdcdd2ecac8 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Mon, 24 Apr 2023 15:27:39 -0400 Subject: [PATCH 032/110] Update invocations of EmitBundle task --- src/mono/msbuild/common/LibraryBuilder.targets | 6 +++--- src/mono/wasi/build/WasiApp.Native.targets | 10 ++++++++-- src/mono/wasi/wasi.proj | 9 +++++++-- src/mono/wasm/wasm.proj | 15 ++++++++++++--- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index a7db86579dd9ae..e02feae92126b3 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -54,6 +54,7 @@ + @@ -77,9 +78,8 @@ - - + OutputDirectory="$(BundleOutputDirectory)" + /> \ No newline at end of file diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index ef40bafc18fe26..673e519a23db41 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -347,6 +347,7 @@ will emit corresponding .o files for anything we don't already have on disk. --> <_WasmAssembliesBundleObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_assemblies.o + <_WasmAssembliesBundlePreallocatedObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_preallocated_assemblies.o @@ -360,19 +361,24 @@ <_WasiObjectFilesForBundle Include="$(_WasmAssembliesBundleObjectFile)" /> + <_WasiObjectFilesForBundle Include="$(_WasmAssembliesBundlePreallocatedObjectFile)" /> <_WasiObjectFilesForBundle Include="%(WasmBundleAssembliesWithHashes.DestinationFile)" /> + + diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index ab57d08283d29f..04df59dc901f53 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -70,6 +70,7 @@ <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) <_WasmTimezonesBundleObjectFile>$(WasiObjDir)\wasm-bundled-timezones.o + <_WasmTimezonesBundlePreallocatedObjectFile>$(WasiObjDir)\wasm-bundled-preallocated-timezones.o <_WasmTimezonesBundleArchive>$(WasiObjDir)\wasm-bundled-timezones.a <_WasmTimezonesArchiveRsp>$(WasiObjDir)\wasm-bundled-timezones-archive.rsp @@ -89,8 +90,10 @@ @@ -109,7 +112,9 @@ <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> + <_WasmBundleTimezonesToDelete Remove="$(WasiObjDir)mono-bundled-source.h" /> <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundlePreallocatedObjectFile)" /> <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index 016107045d4883..599667108c3205 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -86,6 +86,8 @@ <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) <_WasmTimezonesBundleSourceFile>$(WasmObjDir)\wasm-bundled-timezones.c <_WasmTimezonesBundleObjectFile>$(WasmObjDir)\wasm-bundled-timezones.o + <_WasmTimezonesBundlePreallocatedSourceFile>$(WasmObjDir)\wasm-bundled-preallocated-timezones.c + <_WasmTimezonesBundlePreallocatedObjectFile>$(WasmObjDir)\wasm-bundled-preallocated-timezones.o <_WasmTimezonesBundleArchive>$(WasmObjDir)\wasm-bundled-timezones.a <_WasmTimezonesSourcesRsp>$(WasmObjDir)\wasm-bundled-timezones-sources.rsp <_WasmTimezonesArchiveRsp>$(WasmObjDir)\wasm-bundled-timezones-archive.rsp @@ -106,13 +108,16 @@ <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), %(_WasmBundleTimezonesWithHashes.DestinationFile)).Replace('\','/'))" /> <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), $(_WasmTimezonesBundleSourceFile)).Replace('\','/'))" /> + <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), $(_WasmTimezonesBundlePreallocatedSourceFile)).Replace('\','/'))" /> + <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.c" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(_WasmIntermediateOutputPath)mono-bundled-source.h" /> <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleSourceFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundlePreallocatedSourceFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundlePreallocatedObjectFile)" /> <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.ObjectFile)" /> From 46bd2112b0998a59c8d7329dcacdfea0c6f526ca Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Wed, 3 May 2023 15:05:03 -0400 Subject: [PATCH 033/110] Address feedback Migrate logic to check hash table for assemblies or satellite assemblies back to bundled-resources.c Prepend apis with consistent mono_bundled_resources prefix --- src/mono/mono/metadata/assembly-internals.h | 6 --- src/mono/mono/metadata/assembly.c | 32 ++++------- .../metadata/bundled-resources-internals.h | 12 +++-- src/mono/mono/metadata/bundled-resources.c | 54 +++++++++++++++---- src/mono/mono/metadata/mono-debug.c | 4 +- src/mono/wasi/runtime/driver.c | 4 +- src/tasks/EmitBundleTask/EmitBundleBase.cs | 6 +-- .../Templates/mono-bundled-source.h | 14 +++-- src/tasks/LibraryBuilder/Templates/autoinit.c | 4 +- 9 files changed, 81 insertions(+), 55 deletions(-) diff --git a/src/mono/mono/metadata/assembly-internals.h b/src/mono/mono/metadata/assembly-internals.h index 1d3e567b534d17..f76a28fa283236 100644 --- a/src/mono/mono/metadata/assembly-internals.h +++ b/src/mono/mono/metadata/assembly-internals.h @@ -141,12 +141,6 @@ mono_assembly_get_image_internal (MonoAssembly *assembly); MonoBundledAssembly * mono_create_new_bundled_assembly (const char *name, const unsigned char *data, unsigned int size); -void -mono_hash_contains_bundled_assemblies (gboolean contains); - -void -mono_hash_contains_bundled_satellite_assemblies (gboolean contains); - void mono_set_assemblies_path_direct (char **path); diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 6e334fbac5e7e9..f95ab8a0d31b33 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -83,9 +83,7 @@ mono_assemblies_unlock (void) /* If defined, points to the bundled assembly information */ static const MonoBundledAssembly **bundles; -static gboolean hash_contains_bundled_assemblies; static const MonoBundledSatelliteAssembly **satellite_bundles; -static gboolean hash_contains_bundled_satellite_assemblies; /* Class lazy loading functions */ static GENERATE_TRY_GET_CLASS_WITH_CACHE (debuggable_attribute, "System.Diagnostics", "DebuggableAttribute") @@ -712,8 +710,8 @@ mono_assembly_get_assemblyref (MonoImage *image, int index, MonoAssemblyName *an static MonoAssembly * search_bundle_for_assembly (MonoAssemblyLoadContext *alc, MonoAssemblyName *aname) { - if (bundles == NULL && !hash_contains_bundled_assemblies && - satellite_bundles == NULL && !hash_contains_bundled_satellite_assemblies) + if (bundles == NULL && !mono_bundled_resources_contains_assemblies () && + satellite_bundles == NULL && !mono_bundled_resources_contains_satellite_assemblies ()) return NULL; MonoImageOpenStatus status; @@ -796,7 +794,7 @@ netcore_load_reference (MonoAssemblyName *aname, MonoAssemblyLoadContext *alc, M } } - if ((bundles != NULL || hash_contains_bundled_assemblies) && !is_satellite) { + if ((bundles != NULL || mono_bundled_resources_contains_assemblies ()) && !is_satellite) { reference = search_bundle_for_assembly (mono_alc_get_default (), aname); if (reference) { mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "Assembly found in the bundle: '%s'.", aname->name); @@ -804,7 +802,7 @@ netcore_load_reference (MonoAssemblyName *aname, MonoAssemblyLoadContext *alc, M } } - if ((satellite_bundles != NULL || hash_contains_bundled_satellite_assemblies) && is_satellite) { + if ((satellite_bundles != NULL || mono_bundled_resources_contains_satellite_assemblies ()) && is_satellite) { // Satellite assembly byname requests should be loaded in the same ALC as their parent assembly size_t name_len = strlen (aname->name); char *parent_name = NULL; @@ -1475,10 +1473,10 @@ bundled_assembly_match (const char *bundled_name, const char *name) static MonoImage * open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status) { - if (!bundles && !hash_contains_bundled_assemblies) + if (!bundles && !mono_bundled_resources_contains_assemblies ()) return NULL; - MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)mono_get_bundled_resource_data (filename); + MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)mono_bundled_resources_get (filename); if (assembly) return mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, filename, NULL); @@ -1495,13 +1493,13 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M static MonoImage * open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status, const char *culture) { - if (!satellite_bundles && !hash_contains_bundled_satellite_assemblies) + if (!satellite_bundles && !mono_bundled_resources_contains_satellite_assemblies ()) return NULL; MonoImage *image = NULL; char *bundle_name = g_strconcat (culture, "/", filename, (const char *)NULL); - MonoBundledSatelliteAssemblyResource *satellite_assembly = (MonoBundledSatelliteAssemblyResource *)mono_get_bundled_resource_data (filename); + MonoBundledSatelliteAssemblyResource *satellite_assembly = (MonoBundledSatelliteAssemblyResource *)mono_bundled_resources_get (filename); if (satellite_assembly) image = mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, bundle_name, NULL); @@ -1621,7 +1619,7 @@ mono_assembly_request_open (const char *filename, const MonoAssemblyOpenRequest // If VM built with mkbundle loaded_from_bundle = FALSE; - if (bundles != NULL || hash_contains_bundled_assemblies) { + if (bundles != NULL || mono_bundled_resources_contains_assemblies ()) { /* We don't know the culture of the filename we're loading here, so this call is not culture aware. */ image = mono_assembly_open_from_bundle (load_req.alc, fname, status, NULL); loaded_from_bundle = image != NULL; @@ -3188,12 +3186,6 @@ mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) bundles = assemblies; } -void -mono_hash_contains_bundled_assemblies (gboolean contains) -{ - hash_contains_bundled_assemblies = contains; -} - /** * mono_create_new_bundled_satellite_assembly: */ @@ -3219,12 +3211,6 @@ mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly * satellite_bundles = assemblies; } -void -mono_hash_contains_bundled_satellite_assemblies (gboolean contains) -{ - hash_contains_bundled_satellite_assemblies = contains; -} - /** * mono_assembly_is_jit_optimizer_disabled: * diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 6cb1c1359f06d3..3656392383a6ff 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -51,12 +51,18 @@ typedef struct _MonoBundledSatelliteAssemblyResource { } MonoBundledSatelliteAssemblyResource; void -mono_free_bundled_resources (void); +mono_bundled_resources_free (void); void -mono_add_bundled_resource (MonoBundledResource **resources_to_bundle, uint32_t len); +mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); MonoBundledResource * -mono_get_bundled_resource_data (const char *name); +mono_bundled_resources_get (const char *name); + +gboolean +mono_bundled_resources_contains_assemblies (void); + +gboolean +mono_bundled_resources_contains_satellite_assemblies (void); #endif /* __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ */ \ No newline at end of file diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 60098307c9a498..fe7922266856ed 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -3,6 +3,7 @@ // #include +#include #include "bundled-resources-internals.h" #include "assembly-internals.h" @@ -11,27 +12,28 @@ #include static GHashTable *bundled_resources = NULL; +static gboolean bundle_contains_assemblies, bundle_contains_satellite_assemblies = false; //--------------------------------------------------------------------------------------- // -// mono_free_bundled_resources frees all memory allocated for bundled resources. +// mono_bundled_resources_free frees all memory allocated for bundled resources. // It should only be called when the runtime no longer needs access to the data, // most likely to happen during runtime shutdown. // void -mono_free_bundled_resources (void) +mono_bundled_resources_free (void) { g_hash_table_destroy (bundled_resources); bundled_resources = NULL; - mono_hash_contains_bundled_assemblies (FALSE); - mono_hash_contains_bundled_satellite_assemblies (FALSE); + bundle_contains_assemblies = false; + bundle_contains_satellite_assemblies = false; } //--------------------------------------------------------------------------------------- // -// mono_add_bundled_resource handles bundling of many types of resources to circumvent +// mono_bundled_resources_add handles bundling of many types of resources to circumvent // needing to find or have those resources on disk. The MonoBundledResource struct models // the union of information carried by all supported types of resources which are // enumerated in MonoBundledResourceType. @@ -49,7 +51,7 @@ mono_free_bundled_resources (void) // void -mono_add_bundled_resource (MonoBundledResource **resources_to_bundle, uint32_t len) +mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len) { if (!bundled_resources) bundled_resources = g_hash_table_new (g_str_hash, g_str_equal); @@ -81,17 +83,17 @@ mono_add_bundled_resource (MonoBundledResource **resources_to_bundle, uint32_t l } if (assemblyAdded) - mono_hash_contains_bundled_assemblies (assemblyAdded); + bundle_contains_assemblies = true; if (satelliteAssemblyAdded) - mono_hash_contains_bundled_satellite_assemblies (satelliteAssemblyAdded); + bundle_contains_satellite_assemblies = true; } //--------------------------------------------------------------------------------------- // -// mono_get_bundled_resource_data retrieves the pointer of the MonoBundledResource associated +// mono_bundled_resources_get retrieves the pointer of the MonoBundledResource associated // with a key equivalent to the requested resource name. If the requested bundled resource's -// name has been added via mono_add_bundled_resource, a MonoBundled*Resource had been +// name has been added via mono_bundled_resources_add, a MonoBundled*Resource had been // preallocated, typically through EmitBundleTask. // // Arguments: @@ -102,7 +104,37 @@ mono_add_bundled_resource (MonoBundledResource **resources_to_bundle, uint32_t l // MonoBundledResource * -mono_get_bundled_resource_data (const char *name) +mono_bundled_resources_get (const char *name) { return g_hash_table_lookup (bundled_resources, name); } + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_contains_assemblies returns whether or not assemblies +// have been added to the bundled resource hash table via mono_bundled_resources_add. +// +// Returns: +// gboolean - bool value indicating whether or not a bundled assembly resource had been added. +// + +gboolean +mono_bundled_resources_contains_assemblies (void) +{ + return bundle_contains_assemblies; +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_contains_satellite_assemblies returns whether or not satellite assemblies +// have been added to the bundled resource hash table via mono_bundled_resources_add. +// +// Returns: +// gboolean - bool value indicating whether or not a bundled satellite assembly resource had been added. +// + +gboolean +mono_bundled_resources_contains_satellite_assemblies (void) +{ + return bundle_contains_satellite_assemblies; +} diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 8bcd8caad03d2f..ef8a94a742a98b 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1121,7 +1121,7 @@ open_symfile_from_bundle (MonoImage *image) { BundledSymfile *bsymfile; - MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)mono_get_bundled_resource_data (image->module_name); + MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)mono_bundled_resources_get (image->module_name); if (assembly && assembly->symfile.data) return mono_debug_open_image (image, assembly->symfile.data, assembly->symfile.size); @@ -1132,7 +1132,7 @@ open_symfile_from_bundle (MonoImage *image) if (module_name_dll_suffix && !g_strcasecmp (".webcil", &image->module_name [len - 7])) { memcpy (module_name_dll_suffix + len - 7, ".dll", 4); *(module_name_dll_suffix + len - 3) = '\0'; - assembly = (MonoBundledAssemblyResource *)mono_get_bundled_resource_data (module_name_dll_suffix); + assembly = (MonoBundledAssemblyResource *)mono_bundled_resources_get (module_name_dll_suffix); } g_free (module_name_dll_suffix); if (assembly && assembly->symfile) diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index f80457d4a3e290..6a57c5fa3e93d1 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -69,7 +69,7 @@ extern void mono_register_assemblies_bundle(); #ifndef INVARIANT_GLOBALIZATION extern void mono_register_icu_bundle(); #endif /* INVARIANT_GLOBALIZATION */ -extern MonoBundledResource * mono_get_bundled_resource_data (const char *name); +extern MonoBundledResource * mono_bundled_resources_get (const char *name); #endif /* WASM_SINGLE_FILE */ extern const char* dotnet_wasi_getentrypointassemblyname(); @@ -347,7 +347,7 @@ void load_icu_data (void) #ifdef WASM_SINGLE_FILE mono_register_icu_bundle(); - MonoBundledDataResource *icu_data = (MonoBundledDataResource *)mono_get_bundled_resource_data ("icudt.dat"); + MonoBundledDataResource *icu_data = (MonoBundledDataResource *)mono_bundled_resources_get ("icudt.dat"); if (!icu_data) { printf("Could not load icudt.dat from the bundle"); assert(buffer); diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 66204f79077430..30dcedf8d4b1cc 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -298,17 +298,17 @@ private static void GenerateBundledResourcesRegistration(string bundleRegistrati if (assembliesCount != 0) { preallocatedAssemblies.AppendLine("};"); preallocatedResources.AppendLine(preallocatedAssemblies.ToString()); - addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledAssemblyResources, {assembliesCount});"); + addPreallocatedResources.AppendLine($" mono_bundled_resources_add (bundledAssemblyResources, {assembliesCount});"); } if (satelliteAssembliesCount != 0) { preallocatedSatelliteAssemblies.AppendLine("};"); preallocatedResources.AppendLine(preallocatedSatelliteAssemblies.ToString()); - addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledSatelliteAssemblyResources, {satelliteAssembliesCount});"); + addPreallocatedResources.AppendLine($" mono_bundled_resources_add (bundledSatelliteAssemblyResources, {satelliteAssembliesCount});"); } if (dataCount != 0) { preallocatedData.AppendLine("};"); preallocatedResources.AppendLine(preallocatedData.ToString()); - addPreallocatedResources.AppendLine($" mono_add_bundled_resource (bundledDataResources, {dataCount});"); + addPreallocatedResources.AppendLine($" mono_bundled_resources_add (bundledDataResources, {dataCount});"); } outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-resource-registration.c") diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h index 3a73328e1ae6ca..ba7799b5904ec5 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h @@ -50,16 +50,22 @@ typedef struct _MonoBundledSatelliteAssembly { typedef struct _MonoBundledSatelliteAssemblyResource { MonoBundledResource resource; MonoBundledSatelliteAssembly satellite_assembly; - MonoBundledSymfile symfile; + MonoBundledSymbolData symbol_data; } MonoBundledSatelliteAssemblyResource; void -mono_free_bundled_resources (void); +mono_bundled_resources_free (void); void -mono_add_bundled_resource (MonoBundledResource **resources_to_bundle, uint32_t len); +mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); MonoBundledResource * -mono_get_bundled_resource_data (const char *name); +mono_bundled_resources_get (const char *name); + +int32_t +mono_bundled_resources_contains_assemblies (void); + +int32_t +mono_bundled_resources_contains_satellite_assemblies (void); #endif /* __MONO_BUNDLED_SOURCE_H__ */ \ No newline at end of file diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index 1fe544cbe26011..6188ff22f195de 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -19,6 +19,8 @@ #include "library-builder.h" #include "mono-bundled-source.h" +void mono_register_resources_bundle (void); + static void cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) { @@ -37,7 +39,7 @@ initialize_runtimeconfig (const char *bundle_path) LOG_ERROR ("Out of memory.\n"); const char *file_name = "runtimeconfig.bin"; - MonoBundledDataResource *runtimeConfig = (MonoBundledDataResource *)mono_get_bundled_resource_data (file_name); + MonoBundledDataResource *runtimeConfig = (MonoBundledDataResource *)mono_bundled_resources_get (file_name); if (runtimeConfig) { arg->kind = 1; From ddade815bf0c7063a3bdf7b33d635d9c82a2338d Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Wed, 3 May 2023 16:12:50 -0400 Subject: [PATCH 034/110] [mono] Prefer C99 compliant types for partially exposed APIs --- .../metadata/bundled-resources-internals.h | 6 ++++-- src/mono/mono/metadata/bundled-resources.c | 18 +++++++++--------- .../Templates/mono-bundled-source.h | 6 ++++-- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 3656392383a6ff..de2a4b46b2be41 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -5,6 +5,8 @@ #ifndef __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ #define __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ +#include + #include "loader-internals.h" #include @@ -59,10 +61,10 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t MonoBundledResource * mono_bundled_resources_get (const char *name); -gboolean +bool mono_bundled_resources_contains_assemblies (void); -gboolean +bool mono_bundled_resources_contains_satellite_assemblies (void); #endif /* __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ */ \ No newline at end of file diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index fe7922266856ed..5868acc84705e3 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -12,7 +12,7 @@ #include static GHashTable *bundled_resources = NULL; -static gboolean bundle_contains_assemblies, bundle_contains_satellite_assemblies = false; +static bool bundle_contains_assemblies, bundle_contains_satellite_assemblies = false; //--------------------------------------------------------------------------------------- // @@ -56,21 +56,21 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t if (!bundled_resources) bundled_resources = g_hash_table_new (g_str_hash, g_str_equal); - gboolean assemblyAdded, satelliteAssemblyAdded; + bool assemblyAdded, satelliteAssemblyAdded; - for (int i = 0; i < len; ++i) { + for (uint32_t i = 0; i < len; ++i) { MonoBundledResource *resource_to_bundle = (MonoBundledResource *)resources_to_bundle[i]; switch (resource_to_bundle->type) { case MONO_BUNDLED_ASSEMBLY: { MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)resource_to_bundle; g_hash_table_insert (bundled_resources, (gpointer) assembly->assembly.name, assembly); - assemblyAdded = TRUE; + assemblyAdded = true; break; } case MONO_BUNDLED_SATELLITE_ASSEMBLY: { MonoBundledSatelliteAssemblyResource *satellite_assembly = (MonoBundledSatelliteAssemblyResource *)resource_to_bundle; g_hash_table_insert (bundled_resources, (gpointer) satellite_assembly->satellite_assembly.name, satellite_assembly); - satelliteAssemblyAdded = TRUE; + satelliteAssemblyAdded = true; break; } case MONO_BUNDLED_DATA: @@ -115,10 +115,10 @@ mono_bundled_resources_get (const char *name) // have been added to the bundled resource hash table via mono_bundled_resources_add. // // Returns: -// gboolean - bool value indicating whether or not a bundled assembly resource had been added. +// bool - bool value indicating whether or not a bundled assembly resource had been added. // -gboolean +bool mono_bundled_resources_contains_assemblies (void) { return bundle_contains_assemblies; @@ -130,10 +130,10 @@ mono_bundled_resources_contains_assemblies (void) // have been added to the bundled resource hash table via mono_bundled_resources_add. // // Returns: -// gboolean - bool value indicating whether or not a bundled satellite assembly resource had been added. +// bool - bool value indicating whether or not a bundled satellite assembly resource had been added. // -gboolean +bool mono_bundled_resources_contains_satellite_assemblies (void) { return bundle_contains_satellite_assemblies; diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h index ba7799b5904ec5..3fa3da124bfc35 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h @@ -4,6 +4,8 @@ #ifndef __MONO_BUNDLED_SOURCE_H__ #define __MONO_BUNDLED_SOURCE_H__ +#include + #include typedef enum { @@ -62,10 +64,10 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t MonoBundledResource * mono_bundled_resources_get (const char *name); -int32_t +bool mono_bundled_resources_contains_assemblies (void); -int32_t +bool mono_bundled_resources_contains_satellite_assemblies (void); #endif /* __MONO_BUNDLED_SOURCE_H__ */ \ No newline at end of file From 7c0081f05f06a52f700a1efe426c70df49a5690e Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 4 May 2023 15:19:40 -0400 Subject: [PATCH 035/110] Address various fixups Remove unused function Remove unnecessary string duplication Add length checks for comparisons Fix spacing in MonoBundledResource structs Avoid g_hash_table_lookup_extended precondition macro assertion fail --- src/mono/mono/metadata/assembly-internals.h | 3 --- src/mono/mono/metadata/assembly.c | 25 +++---------------- .../metadata/bundled-resources-internals.h | 4 +-- src/mono/mono/metadata/bundled-resources.c | 3 +++ src/mono/mono/metadata/mono-debug.c | 2 +- .../Templates/mono-bundled-source.h | 4 +-- .../Templates/preloaded-assemblies.c | 2 +- 7 files changed, 13 insertions(+), 30 deletions(-) diff --git a/src/mono/mono/metadata/assembly-internals.h b/src/mono/mono/metadata/assembly-internals.h index f76a28fa283236..b5e18fa99dfc15 100644 --- a/src/mono/mono/metadata/assembly-internals.h +++ b/src/mono/mono/metadata/assembly-internals.h @@ -138,9 +138,6 @@ mono_assembly_get_name_internal (MonoAssembly *assembly); MONO_PROFILER_API MonoImage* mono_assembly_get_image_internal (MonoAssembly *assembly); -MonoBundledAssembly * -mono_create_new_bundled_assembly (const char *name, const unsigned char *data, unsigned int size); - void mono_set_assemblies_path_direct (char **path); diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index f95ab8a0d31b33..6b9c3d8f531291 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1532,17 +1532,14 @@ mono_assembly_open_from_bundle (MonoAssemblyLoadContext *alc, const char *filena */ MonoImage *image = NULL; MONO_ENTER_GC_UNSAFE; - gboolean is_satellite = culture && culture [0] != 0; - const char *name; - if (is_satellite) { - name = g_strdup (filename); - image = open_from_satellite_bundle (alc, name, status, culture); + if (culture && culture [0] != 0) { + image = open_from_satellite_bundle (alc, filename, status, culture); } else { - name = g_path_get_basename (filename); + const char *name = g_path_get_basename (filename); image = open_from_bundle_internal (alc, name, status); + g_free ((void *)name); } - g_free ((void *)name); if (image) { mono_image_addref (image); mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "Assembly Loader loaded assembly from bundle: '%s'.", filename); @@ -3163,20 +3160,6 @@ mono_assembly_get_name_internal (MonoAssembly *assembly) return &assembly->aname; } -/** - * mono_create_new_bundled_assembly: - */ -MonoBundledAssembly * -mono_create_new_bundled_assembly (const char *name, const unsigned char *data, unsigned int size) -{ - MonoBundledAssembly *assembly = g_new0 (MonoBundledAssembly, 1); - assembly->name = strdup (name); - g_assert (assembly->name); - assembly->data = data; - assembly->size = size; - return assembly; -} - /** * mono_register_bundled_assemblies: */ diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index de2a4b46b2be41..3ed50422f3d67f 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -43,13 +43,13 @@ typedef struct _MonoBundledSymfile { typedef struct _MonoBundledAssemblyResource { MonoBundledResource resource; MonoBundledAssembly assembly; - MonoBundledSymfile symfile; + MonoBundledSymfile symfile; } MonoBundledAssemblyResource; typedef struct _MonoBundledSatelliteAssemblyResource { MonoBundledResource resource; MonoBundledSatelliteAssembly satellite_assembly; - MonoBundledSymfile symfile; + MonoBundledSymfile symfile; } MonoBundledSatelliteAssemblyResource; void diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 5868acc84705e3..753c422824fa35 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -106,6 +106,9 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t MonoBundledResource * mono_bundled_resources_get (const char *name) { + if (!bundled_resources) + return NULL; + return g_hash_table_lookup (bundled_resources, name); } diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index ef8a94a742a98b..eeac4947f447ba 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1129,7 +1129,7 @@ open_symfile_from_bundle (MonoImage *image) int len = strlen (image->module_name); char *module_name_dll_suffix = strdup (image->module_name); /* if image's module_name ends with .webcil, check if theres a bundled resource with a .dll extension instead */ - if (module_name_dll_suffix && !g_strcasecmp (".webcil", &image->module_name [len - 7])) { + if (module_name_dll_suffix && len >= 7 && !g_strcasecmp (".webcil", &image->module_name [len - 7])) { memcpy (module_name_dll_suffix + len - 7, ".dll", 4); *(module_name_dll_suffix + len - 3) = '\0'; assembly = (MonoBundledAssemblyResource *)mono_bundled_resources_get (module_name_dll_suffix); diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h index 3fa3da124bfc35..6f72fbe59ff7e5 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h @@ -39,7 +39,7 @@ typedef struct _MonoBundledSymfile { typedef struct _MonoBundledAssemblyResource { MonoBundledResource resource; MonoBundledAssembly assembly; - MonoBundledSymfile symfile; + MonoBundledSymfile symfile; } MonoBundledAssemblyResource; typedef struct _MonoBundledSatelliteAssembly { @@ -52,7 +52,7 @@ typedef struct _MonoBundledSatelliteAssembly { typedef struct _MonoBundledSatelliteAssemblyResource { MonoBundledResource resource; MonoBundledSatelliteAssembly satellite_assembly; - MonoBundledSymbolData symbol_data; + MonoBundledSymbolData symbol_data; } MonoBundledSatelliteAssemblyResource; void diff --git a/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c b/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c index 00e6a99c7a9620..f650ad78650c42 100644 --- a/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c +++ b/src/tasks/LibraryBuilder/Templates/preloaded-assemblies.c @@ -18,7 +18,7 @@ preload_assembly (const char* filename) if (!filename_without_extension) LOG_ERROR ("Out of memory.\n"); - if (!STR_CASE_CMP (".dll", &filename [len - 4])) + if (len >= 4 && !STR_CASE_CMP (".dll", &filename [len - 4])) *(filename_without_extension + len - 4) = '\0'; assembly = mono_assembly_load_with_partial_name (filename_without_extension, NULL); From 39b510715337856e1198c9df25d61a50b65ee6b2 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 4 May 2023 13:19:34 -0400 Subject: [PATCH 036/110] Combine preallocation registration and rename to templates --- .../msbuild/common/LibraryBuilder.targets | 2 +- src/tasks/EmitBundleTask/EmitBundleBase.cs | 106 ++++++------------ ...embly.c => mono-bundled-assembly.template} | 0 ...dled-data.c => mono-bundled-data.template} | 0 .../mono-bundled-preallocated-source.c | 6 - ...e-preallocation-and-registration.template} | 2 + ... mono-bundled-satellite-assembly.template} | 0 .../Templates/mono-bundled-source.c | 3 - ...ymfile.c => mono-bundled-symfile.template} | 0 9 files changed, 35 insertions(+), 84 deletions(-) rename src/tasks/EmitBundleTask/Templates/{mono-bundled-assembly.c => mono-bundled-assembly.template} (100%) rename src/tasks/EmitBundleTask/Templates/{mono-bundled-data.c => mono-bundled-data.template} (100%) delete mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c rename src/tasks/EmitBundleTask/Templates/{mono-bundled-resource-registration.c => mono-bundled-resource-preallocation-and-registration.template} (92%) rename src/tasks/EmitBundleTask/Templates/{mono-bundled-satellite-assembly.c => mono-bundled-satellite-assembly.template} (100%) delete mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-source.c rename src/tasks/EmitBundleTask/Templates/{mono-bundled-symfile.c => mono-bundled-symfile.template} (100%) diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index e02feae92126b3..e486beefc7c7d2 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -46,7 +46,7 @@ - mono-bundled-source.c + mono-bundled-resource-data.c $(BundleDir) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 30dcedf8d4b1cc..58dd9f18b70a93 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -16,7 +16,6 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelableTask { private CancellationTokenSource BuildTaskCancelled { get; } = new(); - private static ITaskItem[] PreallocatedSymbols { get; set; } = Array.Empty(); /// Must have DestinationFile metadata, which is the output filename /// Could have RegisteredName, otherwise it would be the filename. @@ -42,12 +41,6 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab /// public string BundleHeader { get; set; } = "mono-bundled-source.h"; - /// - /// The filename for the generated source file that preallocates - /// MonoBundled*Resource structs. - /// - public string BundlePreallocationFile { get; set; } = "mono-bundled-preallocated-source.c"; - /// /// Filename for the unified source containing all byte data and len /// of the FilesToBundle resources. @@ -145,18 +138,11 @@ public override bool Execute() // Generate header containing MonoBundled*Resource typedefs File.WriteAllText(Path.Combine(OutputDirectory, BundleHeader), Utils.GetEmbeddedResource("mono-bundled-source.h")); - // Generate source file containing preallocated MonoBundled*Resource structs - Emit(Path.Combine(OutputDirectory, BundlePreallocationFile), (inputStream) => - { - using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); - GeneratePreallocatedMonoBundleStructs(outputUtf8Writer, files); - }); - - // Generate source file to register bundled resources + // Generate source file to preallocate resources and register bundled resources Emit(Path.Combine(OutputDirectory, BundleFile), (inputStream) => { using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); - GenerateBundledResourcesRegistration(BundleRegistrationFunctionName, outputUtf8Writer); + GenerateBundledResourcePreallocationAndRegistration(BundleRegistrationFunctionName, files, outputUtf8Writer); }); return true; @@ -201,97 +187,68 @@ private static byte[] InitLookupTable() private static Dictionary symbolDataLen = new(); - private static void GeneratePreallocatedMonoBundleStructs(StreamWriter outputUtf8Writer, ICollection<(string registeredName, string symbol, string? symfileSymbol)> files) + private static void GenerateBundledResourcePreallocationAndRegistration(string bundleRegistrationFunctionName, ICollection<(string registeredName, string symbol, string? symfileSymbol)> files, StreamWriter outputUtf8Writer) { - var preallocatedSymbols = new ArrayList(); StringBuilder preallocatedSource = new(); + string assemblyTemplate = Utils.GetEmbeddedResource("mono-bundled-assembly.template"); + string satelliteAssemblyTemplate = Utils.GetEmbeddedResource("mono-bundled-satellite-assembly.template"); + string symbolDataTemplate = Utils.GetEmbeddedResource("mono-bundled-data.template"); + + var preallocatedResources = new StringBuilder(); + var preallocatedAssemblies = new StringBuilder("MonoBundledResource *bundledAssemblyResources[] = { "); + var preallocatedSatelliteAssemblies = new StringBuilder("MonoBundledResource *bundledSatelliteAssemblyResources[] = { "); + var preallocatedData = new StringBuilder("MonoBundledResource *bundledDataResources[] = { "); + int assembliesCount = 0; + int satelliteAssembliesCount = 0; + int dataCount = 0; foreach (var tuple in files) { - var preallocatedSymbol = new TaskItem(tuple.symbol); - // extern symbols - StringBuilder preallocatedData = new(); - preallocatedData.AppendLine($"extern const unsigned char {tuple.symbol}_data[];"); + StringBuilder preallocatedResourceData = new(); + preallocatedResourceData.AppendLine($"extern const unsigned char {tuple.symbol}_data[];"); if (!string.IsNullOrEmpty(tuple.symfileSymbol)) { - preallocatedData.AppendLine($"extern const unsigned char {tuple.symfileSymbol}_data[];"); + preallocatedResourceData.AppendLine($"extern const unsigned char {tuple.symfileSymbol}_data[];"); } - preallocatedSource.AppendLine(preallocatedData.ToString()); + preallocatedSource.AppendLine(preallocatedResourceData.ToString()); // Generate Preloaded MonoBundled*Resource structs string preloadedStruct; switch (GetFileType(tuple.registeredName)) { case "MONO_BUNDLED_ASSEMBLY": { - preloadedStruct = Utils.GetEmbeddedResource("mono-bundled-assembly.c"); - preallocatedSymbol.SetMetadata("ResourceType", "MonoBundledAssemblyResource"); + preloadedStruct = assemblyTemplate; + preallocatedAssemblies.Append($"(MonoBundledResource *)&{tuple.symbol}, "); + assembliesCount += 1; break; } case "MONO_BUNDLED_SATELLITE_ASSEMBLY": { - preloadedStruct = Utils.GetEmbeddedResource("mono-bundled-satellite-assembly.c"); - preallocatedSymbol.SetMetadata("ResourceType", "MonoBundledSatelliteAssemblyResource"); + preloadedStruct = satelliteAssemblyTemplate; + preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{tuple.symbol}, "); + satelliteAssembliesCount += 1; break; } case "MONO_BUNDLED_DATA": default: { - preloadedStruct = Utils.GetEmbeddedResource("mono-bundled-data.c"); - preallocatedSymbol.SetMetadata("ResourceType", "MonoBundledDataResource"); + preloadedStruct = symbolDataTemplate; + preallocatedData.Append($"(MonoBundledResource *)&{tuple.symbol}, "); + dataCount += 1; break; } } + // Add associated symfile information to MonoBundledAssemblyResource/MonoBundleSatelliteAssemblyResource structs string preloadedSymfile = ""; if (!string.IsNullOrEmpty(tuple.symfileSymbol)) { - preloadedSymfile = Utils.GetEmbeddedResource("mono-bundled-symfile.c") + preloadedSymfile = Utils.GetEmbeddedResource("mono-bundled-symfile.template") .Replace("%SymfileSymbol%", tuple.symfileSymbol) .Replace("%SymLen%", symbolDataLen[tuple.symfileSymbol].ToString()); } preallocatedSource.AppendLine(preloadedStruct.Replace("%RegisteredName%", tuple.registeredName) .Replace("%Symbol%", tuple.symbol) .Replace("%Len%", symbolDataLen[tuple.symbol].ToString()) - .Replace("%MonoBundledSymfile%", preloadedSymfile)); - - preallocatedSymbols.Add(preallocatedSymbol); - } - - outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-preallocated-source.c") - .Replace("%PreallocatedStructs%", preallocatedSource.ToString())); - - PreallocatedSymbols = (ITaskItem[])preallocatedSymbols.ToArray(typeof(ITaskItem)); - } - - private static void GenerateBundledResourcesRegistration(string bundleRegistrationFunctionName, StreamWriter outputUtf8Writer) - { - var preallocatedResources = new StringBuilder(); - var preallocatedAssemblies = new StringBuilder("MonoBundledResource *bundledAssemblyResources[] = { "); - var preallocatedSatelliteAssemblies = new StringBuilder("MonoBundledResource *bundledSatelliteAssemblyResources[] = { "); - var preallocatedData = new StringBuilder("MonoBundledResource *bundledDataResources[] = { "); - int assembliesCount = 0; - int satelliteAssembliesCount = 0; - int dataCount = 0; - foreach (ITaskItem preallocatedSymbolItem in PreallocatedSymbols) - { - preallocatedResources.AppendLine($"extern const {preallocatedSymbolItem.GetMetadata("ResourceType")} {preallocatedSymbolItem.ItemSpec};"); - - switch (preallocatedSymbolItem.GetMetadata("ResourceType")) { - case "MonoBundledAssemblyResource": { - preallocatedAssemblies.Append($"(MonoBundledResource *)&{preallocatedSymbolItem.ItemSpec}, "); - assembliesCount += 1; - break; - } - case "MonoBundledSatelliteAssemblyResource": { - preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{preallocatedSymbolItem.ItemSpec}, "); - satelliteAssembliesCount += 1; - break; - } - case "MonoBundledDataResource": - default: { - preallocatedData.Append($"(MonoBundledResource *)&{preallocatedSymbolItem.ItemSpec}, "); - dataCount += 1; - break; - } - } + .Replace("%MonoBundledSymbolData%", preloadedSymfile)); } var addPreallocatedResources = new StringBuilder(); @@ -311,7 +268,8 @@ private static void GenerateBundledResourcesRegistration(string bundleRegistrati addPreallocatedResources.AppendLine($" mono_bundled_resources_add (bundledDataResources, {dataCount});"); } - outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-resource-registration.c") + outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-resource-preallocation-and-registration.template") + .Replace("%PreallocatedStructs%", preallocatedSource.ToString()) .Replace("%PreallocatedResources%", preallocatedResources.ToString()) .Replace("%BundleRegistrationFunctionName%", bundleRegistrationFunctionName) .Replace("%AddPreallocatedResources%", addPreallocatedResources.ToString())); diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template similarity index 100% rename from src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.c rename to src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-data.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template similarity index 100% rename from src/tasks/EmitBundleTask/Templates/mono-bundled-data.c rename to src/tasks/EmitBundleTask/Templates/mono-bundled-data.template diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c deleted file mode 100644 index b312e5c1084196..00000000000000 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-preallocated-source.c +++ /dev/null @@ -1,6 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#include "mono-bundled-source.h" - -%PreallocatedStructs% \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-registration.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template similarity index 92% rename from src/tasks/EmitBundleTask/Templates/mono-bundled-resource-registration.c rename to src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template index b9ba67f58761d1..004c5ddac5c52e 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-registration.c +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template @@ -3,6 +3,8 @@ #include "mono-bundled-source.h" +%PreallocatedStructs% + %PreallocatedResources% void diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template similarity index 100% rename from src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.c rename to src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.c deleted file mode 100644 index f233f035347251..00000000000000 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.c +++ /dev/null @@ -1,3 +0,0 @@ -unsigned char %Symbol%_data[] = {%Data% 0 -}; -unsigned int %Symbol%_len = %Length%; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.c b/src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.template similarity index 100% rename from src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.c rename to src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.template From 69af44c5828bd93baa2397f191c9786cff37cbe0 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 4 May 2023 15:24:25 -0400 Subject: [PATCH 037/110] Prefer symbol data over symfile --- .../metadata/bundled-resources-internals.h | 8 ++++---- src/mono/mono/metadata/mono-debug.c | 4 ++-- src/tasks/EmitBundleTask/EmitBundleBase.cs | 18 +++++++++--------- .../Templates/mono-bundled-assembly.template | 2 +- .../mono-bundled-satellite-assembly.template | 2 +- .../Templates/mono-bundled-source.h | 6 +++--- .../Templates/mono-bundled-symbol.template | 2 ++ .../Templates/mono-bundled-symfile.template | 2 -- 8 files changed, 22 insertions(+), 22 deletions(-) create mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-symbol.template delete mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.template diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 3ed50422f3d67f..4620abd4348460 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -35,21 +35,21 @@ typedef struct _MonoBundledDataResource { MonoBundledData data; } MonoBundledDataResource; -typedef struct _MonoBundledSymfile { +typedef struct _MonoBundledSymbolData { const unsigned char *data; unsigned int size; -} MonoBundledSymfile; +} MonoBundledSymbolData; typedef struct _MonoBundledAssemblyResource { MonoBundledResource resource; MonoBundledAssembly assembly; - MonoBundledSymfile symfile; + MonoBundledSymbolData symbol_data; } MonoBundledAssemblyResource; typedef struct _MonoBundledSatelliteAssemblyResource { MonoBundledResource resource; MonoBundledSatelliteAssembly satellite_assembly; - MonoBundledSymfile symfile; + MonoBundledSymbolData symbol_data; } MonoBundledSatelliteAssemblyResource; void diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index eeac4947f447ba..371c5555309e29 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1122,8 +1122,8 @@ open_symfile_from_bundle (MonoImage *image) BundledSymfile *bsymfile; MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)mono_bundled_resources_get (image->module_name); - if (assembly && assembly->symfile.data) - return mono_debug_open_image (image, assembly->symfile.data, assembly->symfile.size); + if (assembly && assembly->symbol_data.data) + return mono_debug_open_image (image, assembly->symbol_data.data, assembly->symbol_data.size); #ifdef ENABLE_WEBCIL int len = strlen (image->module_name); diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 58dd9f18b70a93..5f70209af29212 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -81,12 +81,12 @@ public override bool Execute() var files = filesToBundleByRegisteredName.Select(group => { var registeredFile = group.First(); var outputFile = registeredFile.GetMetadata("DestinationFile"); - var registeredName = group.Key; - var symbolName = ToSafeSymbolName(outputFile); - string? symfileSymbolName = null; - if (File.Exists(registeredFile.GetMetadata("Symfile"))) - symfileSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("Symfile")); - return (registeredName, symbolName, symfileSymbolName); + var registeredFilename = group.Key; + var resourceName = ToSafeSymbolName(outputFile); + string? resourceSymbolName = null; + if (File.Exists(registeredFile.GetMetadata("SymbolFile"))) + resourceSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("SymbolFile")); + return (registeredFilename, resourceName, resourceSymbolName); }).ToList(); Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); @@ -241,9 +241,9 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b string preloadedSymfile = ""; if (!string.IsNullOrEmpty(tuple.symfileSymbol)) { - preloadedSymfile = Utils.GetEmbeddedResource("mono-bundled-symfile.template") - .Replace("%SymfileSymbol%", tuple.symfileSymbol) - .Replace("%SymLen%", symbolDataLen[tuple.symfileSymbol].ToString()); + preloadedSymfile = Utils.GetEmbeddedResource("mono-bundled-symbol.template") + .Replace("%ResourceSymbolName%", tuple.resourceSymbolName) + .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName].ToString()); } preallocatedSource.AppendLine(preloadedStruct.Replace("%RegisteredName%", tuple.registeredName) .Replace("%Symbol%", tuple.symbol) diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template index 941b9f93822325..b528220c00bef1 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template @@ -3,4 +3,4 @@ const MonoBundledAssemblyResource %Symbol% = { .assembly = { .name = "%RegisteredName%", .data = %Symbol%_data, .size = %Len% }, -%MonoBundledSymfile%}; \ No newline at end of file +%MonoBundledSymbolData%}; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template index deb208bc2fa9d9..b72fa3c598ce5c 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template @@ -3,4 +3,4 @@ const MonoBundledSatelliteAssemblyResource %Symbol% = { .satellite_assembly = { .name = "%RegisteredName%", .data = %Symbol%_data, .size = %Len% }, -%MonoBundledSymfile%}; \ No newline at end of file +%MonoBundledSymbolData%}; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h index 6f72fbe59ff7e5..6ca3567896c2ae 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h @@ -31,15 +31,15 @@ typedef struct _MonoBundledDataResource { MonoBundledData data; } MonoBundledDataResource; -typedef struct _MonoBundledSymfile { +typedef struct _MonoBundledSymbolData { const unsigned char *data; unsigned int size; -} MonoBundledSymfile; +} MonoBundledSymbolData; typedef struct _MonoBundledAssemblyResource { MonoBundledResource resource; MonoBundledAssembly assembly; - MonoBundledSymfile symfile; + MonoBundledSymbolData symbol_data; } MonoBundledAssemblyResource; typedef struct _MonoBundledSatelliteAssembly { diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-symbol.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-symbol.template new file mode 100644 index 00000000000000..03384189fa21c2 --- /dev/null +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-symbol.template @@ -0,0 +1,2 @@ + .symbol_data = { .data = &%SymfileSymbol%_data, + .size = %SymLen% } diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.template deleted file mode 100644 index 640b74ea138b62..00000000000000 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-symfile.template +++ /dev/null @@ -1,2 +0,0 @@ - .symfile = { .data = &%SymfileSymbol%_data, - .size = %SymLen% } From c3401df1a36281b3010d3bd5d5562e528dbff492 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 4 May 2023 14:53:25 -0400 Subject: [PATCH 038/110] Differentiate resource name and symbol --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 29 +++++++++++-------- .../Templates/mono-bundled-assembly.template | 6 ++-- .../Templates/mono-bundled-data.template | 6 ++-- .../mono-bundled-satellite-assembly.template | 6 ++-- .../Templates/mono-bundled-symbol.template | 4 +-- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 5f70209af29212..3ab84b16eef3cd 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -84,8 +84,13 @@ public override bool Execute() var registeredFilename = group.Key; var resourceName = ToSafeSymbolName(outputFile); string? resourceSymbolName = null; +<<<<<<< HEAD if (File.Exists(registeredFile.GetMetadata("SymbolFile"))) resourceSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("SymbolFile")); +======= + if (File.Exists(registeredFile.GetMetadata("Symfile"))) + resourceSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("Symfile")); +>>>>>>> 5a56c0e2b21 (Differentiate resource name and symbol) return (registeredFilename, resourceName, resourceSymbolName); }).ToList(); @@ -187,7 +192,7 @@ private static byte[] InitLookupTable() private static Dictionary symbolDataLen = new(); - private static void GenerateBundledResourcePreallocationAndRegistration(string bundleRegistrationFunctionName, ICollection<(string registeredName, string symbol, string? symfileSymbol)> files, StreamWriter outputUtf8Writer) + private static void GenerateBundledResourcePreallocationAndRegistration(string bundleRegistrationFunctionName, ICollection<(string registeredFilename, string resourceName, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) { StringBuilder preallocatedSource = new(); @@ -206,32 +211,32 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b { // extern symbols StringBuilder preallocatedResourceData = new(); - preallocatedResourceData.AppendLine($"extern const unsigned char {tuple.symbol}_data[];"); - if (!string.IsNullOrEmpty(tuple.symfileSymbol)) + preallocatedResourceData.AppendLine($"extern const unsigned char {tuple.resourceName}_data[];"); + if (!string.IsNullOrEmpty(tuple.resourceSymbolName)) { - preallocatedResourceData.AppendLine($"extern const unsigned char {tuple.symfileSymbol}_data[];"); + preallocatedResourceData.AppendLine($"extern const unsigned char {tuple.resourceSymbolName}_data[];"); } preallocatedSource.AppendLine(preallocatedResourceData.ToString()); // Generate Preloaded MonoBundled*Resource structs string preloadedStruct; - switch (GetFileType(tuple.registeredName)) { + switch (GetFileType(tuple.registeredFilename)) { case "MONO_BUNDLED_ASSEMBLY": { preloadedStruct = assemblyTemplate; - preallocatedAssemblies.Append($"(MonoBundledResource *)&{tuple.symbol}, "); + preallocatedAssemblies.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); assembliesCount += 1; break; } case "MONO_BUNDLED_SATELLITE_ASSEMBLY": { preloadedStruct = satelliteAssemblyTemplate; - preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{tuple.symbol}, "); + preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); satelliteAssembliesCount += 1; break; } case "MONO_BUNDLED_DATA": default: { preloadedStruct = symbolDataTemplate; - preallocatedData.Append($"(MonoBundledResource *)&{tuple.symbol}, "); + preallocatedData.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); dataCount += 1; break; } @@ -239,15 +244,15 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b // Add associated symfile information to MonoBundledAssemblyResource/MonoBundleSatelliteAssemblyResource structs string preloadedSymfile = ""; - if (!string.IsNullOrEmpty(tuple.symfileSymbol)) + if (!string.IsNullOrEmpty(tuple.resourceSymbolName)) { preloadedSymfile = Utils.GetEmbeddedResource("mono-bundled-symbol.template") .Replace("%ResourceSymbolName%", tuple.resourceSymbolName) .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName].ToString()); } - preallocatedSource.AppendLine(preloadedStruct.Replace("%RegisteredName%", tuple.registeredName) - .Replace("%Symbol%", tuple.symbol) - .Replace("%Len%", symbolDataLen[tuple.symbol].ToString()) + preallocatedSource.AppendLine(preloadedStruct.Replace("%RegisteredFilename%", tuple.registeredFilename) + .Replace("%ResourceName%", tuple.resourceName) + .Replace("%Len%", symbolDataLen[tuple.resourceName].ToString()) .Replace("%MonoBundledSymbolData%", preloadedSymfile)); } diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template index b528220c00bef1..502106833c8e69 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template @@ -1,6 +1,6 @@ -const MonoBundledAssemblyResource %Symbol% = { +const MonoBundledAssemblyResource %ResourceName% = { .resource = { .type = MONO_BUNDLED_ASSEMBLY }, - .assembly = { .name = "%RegisteredName%", - .data = %Symbol%_data, + .assembly = { .name = "%RegisteredFilename%", + .data = %ResourceName%_data, .size = %Len% }, %MonoBundledSymbolData%}; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template index 04cbe7b9ea03ac..abd5bcf8801e03 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template @@ -1,6 +1,6 @@ -const MonoBundledDataResource %Symbol% = { +const MonoBundledDataResource %ResourceName% = { .resource = { .type = MONO_BUNDLED_DATA }, - .data = { .name = "%RegisteredName%", - .data = %Symbol%_data, + .data = { .name = "%RegisteredFilename%", + .data = %ResourceName%_data, .size = %Len% }, }; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template index b72fa3c598ce5c..725208a2d84981 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template @@ -1,6 +1,6 @@ -const MonoBundledSatelliteAssemblyResource %Symbol% = { +const MonoBundledSatelliteAssemblyResource %ResourceName% = { .resource = { .type = MONO_BUNDLED_SATELLITE_ASSEMBLY }, - .satellite_assembly = { .name = "%RegisteredName%", - .data = %Symbol%_data, + .satellite_assembly = { .name = "%RegisteredFilename%", + .data = %ResourceName%_data, .size = %Len% }, %MonoBundledSymbolData%}; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-symbol.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-symbol.template index 03384189fa21c2..a7677ec6e270b8 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-symbol.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-symbol.template @@ -1,2 +1,2 @@ - .symbol_data = { .data = &%SymfileSymbol%_data, - .size = %SymLen% } + .symbol_data = { .data = &%ResourceSymbolName%_data, + .size = %SymbolLen% } From 239c64947e34c9a05a5f77af5ed939517c2e23d4 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 4 May 2023 14:30:00 -0400 Subject: [PATCH 039/110] Update invocations of EmitBundle --- src/mono/msbuild/common/LibraryBuilder.targets | 13 +++++++------ src/mono/wasi/build/WasiApp.Native.targets | 8 -------- src/mono/wasi/wasi.proj | 3 --- src/mono/wasm/wasm.proj | 7 ------- src/tasks/EmitBundleTask/EmitBundleBase.cs | 2 +- 5 files changed, 8 insertions(+), 25 deletions(-) diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index e486beefc7c7d2..8b912954062402 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -46,15 +46,15 @@ - mono-bundled-resource-data.c + mono-bundled-resource-data.c + mono-bundled-source.c $(BundleDir) - + + - - @@ -73,11 +73,12 @@ - + diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 756b5d15c0d7df..0df7aaf6d725e1 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -350,9 +350,7 @@ will emit corresponding .o files for anything we don't already have on disk. --> <_WasmAssembliesBundleObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_assemblies.o - <_WasmAssembliesBundlePreallocatedObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_preallocated_assemblies.o <_WasmIcuBundleObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_icu.o - <_WasmIcuBundlePreallocatedObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_preallocated_icu.o @@ -374,13 +372,11 @@ ClangExecutable="$(WasiClang)" BundleName="mono_register_assemblies_bundle" BundleFile="wasi_bundled_assemblies.o" - BundlePreallocationFile="wasi_bundled_preallocated_assemblies.o" OutputDirectory="$(_WasmIntermediateOutputPath)" /> <_WasiObjectFilesForBundle Include="$(_WasmAssembliesBundleObjectFile)" /> - <_WasiObjectFilesForBundle Include="$(_WasmAssembliesBundlePreallocatedObjectFile)" /> <_WasiObjectFilesForBundle Include="%(WasmBundleAssembliesWithHashes.DestinationFile)" /> @@ -389,7 +385,6 @@ - @@ -399,19 +394,16 @@ ClangExecutable="$(WasiClang)" BundleName="mono_register_icu_bundle" BundleFile="wasi_bundled_icu.o" - BundlePreallocationFile="wasi_bundled_preallocated_icu.o" OutputDirectory="$(_WasmIntermediateOutputPath)" /> <_WasiObjectFilesForBundle Include="$(_WasmIcuBundleObjectFile)" /> - <_WasiObjectFilesForBundle Include="$(_WasmIcuBundlePreallocatedObjectFile)" /> <_WasiObjectFilesForBundle Include="%(WasmBundleIcuWithHashes.DestinationFile)" /> - diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index 04df59dc901f53..76db6961766a68 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -70,7 +70,6 @@ <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) <_WasmTimezonesBundleObjectFile>$(WasiObjDir)\wasm-bundled-timezones.o - <_WasmTimezonesBundlePreallocatedObjectFile>$(WasiObjDir)\wasm-bundled-preallocated-timezones.o <_WasmTimezonesBundleArchive>$(WasiObjDir)\wasm-bundled-timezones.a <_WasmTimezonesArchiveRsp>$(WasiObjDir)\wasm-bundled-timezones-archive.rsp @@ -92,7 +91,6 @@ ClangExecutable="$(WasiClang)" BundleName="mono_register_timezones_bundle" BundleFile="wasm-bundled-timezones.o" - BundlePreallocationFile="wasm-bundled-preallocated-timezones.o" OutputDirectory="$(WasiObjDir)" /> @@ -113,7 +111,6 @@ <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> <_WasmBundleTimezonesToDelete Remove="$(WasiObjDir)mono-bundled-source.h" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundlePreallocatedObjectFile)" /> <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index 9fcf2bc22aa608..db2a348782184e 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -86,8 +86,6 @@ <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) <_WasmTimezonesBundleSourceFile>$(WasmObjDir)\wasm-bundled-timezones.c <_WasmTimezonesBundleObjectFile>$(WasmObjDir)\wasm-bundled-timezones.o - <_WasmTimezonesBundlePreallocatedSourceFile>$(WasmObjDir)\wasm-bundled-preallocated-timezones.c - <_WasmTimezonesBundlePreallocatedObjectFile>$(WasmObjDir)\wasm-bundled-preallocated-timezones.o <_WasmTimezonesBundleArchive>$(WasmObjDir)\wasm-bundled-timezones.a <_WasmTimezonesSourcesRsp>$(WasmObjDir)\wasm-bundled-timezones-sources.rsp <_WasmTimezonesArchiveRsp>$(WasmObjDir)\wasm-bundled-timezones-archive.rsp @@ -110,14 +108,12 @@ FilesToBundle="@(_WasmBundleTimezonesWithHashes)" BundleRegistrationFunctionName="mono_register_timezones_bundle" BundleFile="wasm-bundled-timezones.c" - BundlePreallocationFile="wasm-bundled-preallocated-timezones.c" OutputDirectory="$(WasmObjDir)" /> <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), %(_WasmBundleTimezonesWithHashes.DestinationFile)).Replace('\','/'))" /> <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), $(_WasmTimezonesBundleSourceFile)).Replace('\','/'))" /> - <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), $(_WasmTimezonesBundlePreallocatedSourceFile)).Replace('\','/'))" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleSourceFile)" /> <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundlePreallocatedSourceFile)" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundlePreallocatedObjectFile)" /> <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.ObjectFile)" /> diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 3ab84b16eef3cd..8ee05bf719b0e6 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -33,7 +33,7 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab /// The filename for the generated source file that registers /// the bundled resources. /// - public string BundleFile { get; set; } = "mono-bundled-resource-registration.c"; + public string BundleFile { get; set; } = "mono-bundled-source.c"; /// /// The filename for the generated header file that declares From ae088f2a2e0ade6f44e78110db55394f690c1666 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 4 May 2023 16:56:51 -0400 Subject: [PATCH 040/110] Add inlined resource getters --- src/mono/mono/metadata/assembly.c | 4 +-- .../metadata/bundled-resources-internals.h | 34 +++++++++++++++++++ src/mono/mono/metadata/mono-debug.c | 4 +-- src/mono/wasi/runtime/driver.c | 3 +- .../Templates/mono-bundled-source.h | 34 +++++++++++++++++++ src/tasks/LibraryBuilder/Templates/autoinit.c | 2 +- 6 files changed, 74 insertions(+), 7 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 6b9c3d8f531291..2197d23c151513 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1476,7 +1476,7 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M if (!bundles && !mono_bundled_resources_contains_assemblies ()) return NULL; - MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)mono_bundled_resources_get (filename); + MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (filename); if (assembly) return mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, filename, NULL); @@ -1499,7 +1499,7 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, MonoImage *image = NULL; char *bundle_name = g_strconcat (culture, "/", filename, (const char *)NULL); - MonoBundledSatelliteAssemblyResource *satellite_assembly = (MonoBundledSatelliteAssemblyResource *)mono_bundled_resources_get (filename); + MonoBundledSatelliteAssemblyResource *satellite_assembly = mono_bundled_resources_get_satellite_assembly_resource (filename); if (satellite_assembly) image = mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, bundle_name, NULL); diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 4620abd4348460..57976015ff490d 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -5,6 +5,7 @@ #ifndef __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ #define __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ +#include #include #include "loader-internals.h" @@ -61,6 +62,39 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t MonoBundledResource * mono_bundled_resources_get (const char *name); +static inline MonoBundledAssemblyResource * +mono_bundled_resources_get_assembly_resource (const char *name) +{ + MonoBundledAssemblyResource *assembly = + (MonoBundledAssemblyResource*)mono_bundled_resources_get (name); + if (!assembly) + return NULL; + assert (assembly->resource.type == MONO_BUNDLED_ASSEMBLY); + return assembly; +} + +static inline MonoBundledSatelliteAssemblyResource * +mono_bundled_resources_get_satellite_assembly_resource (const char *name) +{ + MonoBundledSatelliteAssemblyResource *satellite_assembly = + (MonoBundledSatelliteAssemblyResource*)mono_bundled_resources_get (name); + if (!satellite_assembly) + return NULL; + assert (satellite_assembly->resource.type == MONO_BUNDLED_SATELLITE_ASSEMBLY); + return satellite_assembly; +} + +static inline MonoBundledDataResource * +mono_bundled_resources_get_data_resource (const char *name) +{ + MonoBundledDataResource *data = + (MonoBundledDataResource*)mono_bundled_resources_get (name); + if (!data) + return NULL; + assert (data->resource.type == MONO_BUNDLED_DATA); + return data; +} + bool mono_bundled_resources_contains_assemblies (void); diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 371c5555309e29..1c7a256ef15b06 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1121,7 +1121,7 @@ open_symfile_from_bundle (MonoImage *image) { BundledSymfile *bsymfile; - MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)mono_bundled_resources_get (image->module_name); + MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (image->module_name); if (assembly && assembly->symbol_data.data) return mono_debug_open_image (image, assembly->symbol_data.data, assembly->symbol_data.size); @@ -1132,7 +1132,7 @@ open_symfile_from_bundle (MonoImage *image) if (module_name_dll_suffix && len >= 7 && !g_strcasecmp (".webcil", &image->module_name [len - 7])) { memcpy (module_name_dll_suffix + len - 7, ".dll", 4); *(module_name_dll_suffix + len - 3) = '\0'; - assembly = (MonoBundledAssemblyResource *)mono_bundled_resources_get (module_name_dll_suffix); + assembly = mono_bundled_resources_get_assembly_resource (module_name_dll_suffix); } g_free (module_name_dll_suffix); if (assembly && assembly->symfile) diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index 6a57c5fa3e93d1..b29d39995bedc2 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -69,7 +69,6 @@ extern void mono_register_assemblies_bundle(); #ifndef INVARIANT_GLOBALIZATION extern void mono_register_icu_bundle(); #endif /* INVARIANT_GLOBALIZATION */ -extern MonoBundledResource * mono_bundled_resources_get (const char *name); #endif /* WASM_SINGLE_FILE */ extern const char* dotnet_wasi_getentrypointassemblyname(); @@ -347,7 +346,7 @@ void load_icu_data (void) #ifdef WASM_SINGLE_FILE mono_register_icu_bundle(); - MonoBundledDataResource *icu_data = (MonoBundledDataResource *)mono_bundled_resources_get ("icudt.dat"); + MonoBundledDataResource *icu_data = mono_bundled_resources_get_data_resource ("icudt.dat"); if (!icu_data) { printf("Could not load icudt.dat from the bundle"); assert(buffer); diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h index 6ca3567896c2ae..0ade0dc19d7d71 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h @@ -4,6 +4,7 @@ #ifndef __MONO_BUNDLED_SOURCE_H__ #define __MONO_BUNDLED_SOURCE_H__ +#include #include #include @@ -64,6 +65,39 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t MonoBundledResource * mono_bundled_resources_get (const char *name); +static inline MonoBundledAssemblyResource * +mono_bundled_resources_get_assembly_resource (const char *name) +{ + MonoBundledAssemblyResource *assembly = + (MonoBundledAssemblyResource*)mono_bundled_resources_get (name); + if (!assembly) + return NULL; + assert (assembly->resource.type == MONO_BUNDLED_ASSEMBLY); + return assembly; +} + +static inline MonoBundledSatelliteAssemblyResource * +mono_bundled_resources_get_satellite_assembly_resource (const char *name) +{ + MonoBundledSatelliteAssemblyResource *satellite_assembly = + (MonoBundledSatelliteAssemblyResource*)mono_bundled_resources_get (name); + if (!satellite_assembly) + return NULL; + assert (satellite_assembly->resource.type == MONO_BUNDLED_SATELLITE_ASSEMBLY); + return satellite_assembly; +} + +static inline MonoBundledDataResource * +mono_bundled_resources_get_data_resource (const char *name) +{ + MonoBundledDataResource *data = + (MonoBundledDataResource*)mono_bundled_resources_get (name); + if (!data) + return NULL; + assert (data->resource.type == MONO_BUNDLED_DATA); + return data; +} + bool mono_bundled_resources_contains_assemblies (void); diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index 6188ff22f195de..8f22058330db5e 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -39,7 +39,7 @@ initialize_runtimeconfig (const char *bundle_path) LOG_ERROR ("Out of memory.\n"); const char *file_name = "runtimeconfig.bin"; - MonoBundledDataResource *runtimeConfig = (MonoBundledDataResource *)mono_bundled_resources_get (file_name); + MonoBundledDataResource *runtimeConfig = mono_bundled_resources_get_data_resource (file_name); if (runtimeConfig) { arg->kind = 1; From 233025ad05f2a2f5f81935a0d8a16685bb7dc8b5 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 5 May 2023 13:58:10 -0400 Subject: [PATCH 041/110] Add unique id field for each MonoBundledResource --- .../mono/metadata/bundled-resources-internals.h | 15 ++++++++------- src/mono/mono/metadata/bundled-resources.c | 16 ++++++++-------- src/tasks/EmitBundleTask/EmitBundleBase.cs | 8 ++++++-- .../Templates/mono-bundled-assembly.template | 3 ++- .../Templates/mono-bundled-data.template | 3 ++- .../mono-bundled-satellite-assembly.template | 3 ++- .../Templates/mono-bundled-source.h | 1 + 7 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 57976015ff490d..fdb70a75ae0dbf 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -22,6 +22,7 @@ typedef enum { typedef struct _MonoBundledResource { MonoBundledResourceType type; + const char *id; void (*free_bundled_resource_func)(void *); } MonoBundledResource; @@ -60,13 +61,13 @@ void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); MonoBundledResource * -mono_bundled_resources_get (const char *name); +mono_bundled_resources_get (const char *id); static inline MonoBundledAssemblyResource * -mono_bundled_resources_get_assembly_resource (const char *name) +mono_bundled_resources_get_assembly_resource (const char *id) { MonoBundledAssemblyResource *assembly = - (MonoBundledAssemblyResource*)mono_bundled_resources_get (name); + (MonoBundledAssemblyResource*)mono_bundled_resources_get (id); if (!assembly) return NULL; assert (assembly->resource.type == MONO_BUNDLED_ASSEMBLY); @@ -74,10 +75,10 @@ mono_bundled_resources_get_assembly_resource (const char *name) } static inline MonoBundledSatelliteAssemblyResource * -mono_bundled_resources_get_satellite_assembly_resource (const char *name) +mono_bundled_resources_get_satellite_assembly_resource (const char *id) { MonoBundledSatelliteAssemblyResource *satellite_assembly = - (MonoBundledSatelliteAssemblyResource*)mono_bundled_resources_get (name); + (MonoBundledSatelliteAssemblyResource*)mono_bundled_resources_get (id); if (!satellite_assembly) return NULL; assert (satellite_assembly->resource.type == MONO_BUNDLED_SATELLITE_ASSEMBLY); @@ -85,10 +86,10 @@ mono_bundled_resources_get_satellite_assembly_resource (const char *name) } static inline MonoBundledDataResource * -mono_bundled_resources_get_data_resource (const char *name) +mono_bundled_resources_get_data_resource (const char *id) { MonoBundledDataResource *data = - (MonoBundledDataResource*)mono_bundled_resources_get (name); + (MonoBundledDataResource*)mono_bundled_resources_get (id); if (!data) return NULL; assert (data->resource.type == MONO_BUNDLED_DATA); diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 753c422824fa35..8d9c24592d43ed 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -63,20 +63,20 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t switch (resource_to_bundle->type) { case MONO_BUNDLED_ASSEMBLY: { MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)resource_to_bundle; - g_hash_table_insert (bundled_resources, (gpointer) assembly->assembly.name, assembly); + g_hash_table_insert (bundled_resources, (gpointer) assembly->resource.id, assembly); assemblyAdded = true; break; } case MONO_BUNDLED_SATELLITE_ASSEMBLY: { MonoBundledSatelliteAssemblyResource *satellite_assembly = (MonoBundledSatelliteAssemblyResource *)resource_to_bundle; - g_hash_table_insert (bundled_resources, (gpointer) satellite_assembly->satellite_assembly.name, satellite_assembly); + g_hash_table_insert (bundled_resources, (gpointer) satellite_assembly->resource.id, satellite_assembly); satelliteAssemblyAdded = true; break; } case MONO_BUNDLED_DATA: default: { MonoBundledDataResource *data = (MonoBundledDataResource *)resource_to_bundle; - g_hash_table_insert (bundled_resources, (gpointer) data->data.name, data); + g_hash_table_insert (bundled_resources, (gpointer) data->resource.id, data); break; } } @@ -92,24 +92,24 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t //--------------------------------------------------------------------------------------- // // mono_bundled_resources_get retrieves the pointer of the MonoBundledResource associated -// with a key equivalent to the requested resource name. If the requested bundled resource's +// with a key equivalent to the requested resource id. If the requested bundled resource's // name has been added via mono_bundled_resources_add, a MonoBundled*Resource had been // preallocated, typically through EmitBundleTask. // // Arguments: -// * name - Unique name of the resource +// * id - Unique name of the resource // // Returns: -// MonoBundledResource * - Pointer to the resource in the hashmap with the key `name` +// MonoBundledResource * - Pointer to the resource in the hashmap with the key `id` // MonoBundledResource * -mono_bundled_resources_get (const char *name) +mono_bundled_resources_get (const char *id) { if (!bundled_resources) return NULL; - return g_hash_table_lookup (bundled_resources, name); + return g_hash_table_lookup (bundled_resources, id); } //--------------------------------------------------------------------------------------- diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 8ee05bf719b0e6..72bc3d959450e2 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -209,6 +209,8 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b int dataCount = 0; foreach (var tuple in files) { + string resourceId = tuple.registeredFilename; + // extern symbols StringBuilder preallocatedResourceData = new(); preallocatedResourceData.AppendLine($"extern const unsigned char {tuple.resourceName}_data[];"); @@ -229,6 +231,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b } case "MONO_BUNDLED_SATELLITE_ASSEMBLY": { preloadedStruct = satelliteAssemblyTemplate; + resourceId = $"{tuple.culture}/{tuple.registeredFilename}"; preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); satelliteAssembliesCount += 1; break; @@ -250,8 +253,9 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b .Replace("%ResourceSymbolName%", tuple.resourceSymbolName) .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName].ToString()); } - preallocatedSource.AppendLine(preloadedStruct.Replace("%RegisteredFilename%", tuple.registeredFilename) - .Replace("%ResourceName%", tuple.resourceName) + preallocatedSource.AppendLine(preloadedStruct.Replace("%ResourceName%", tuple.resourceName) + .Replace("%ResourceID%", resourceId) + .Replace("%RegisteredFilename%", tuple.registeredFilename) .Replace("%Len%", symbolDataLen[tuple.resourceName].ToString()) .Replace("%MonoBundledSymbolData%", preloadedSymfile)); } diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template index 502106833c8e69..88b17a21d4f688 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template @@ -1,5 +1,6 @@ const MonoBundledAssemblyResource %ResourceName% = { - .resource = { .type = MONO_BUNDLED_ASSEMBLY }, + .resource = { .type = MONO_BUNDLED_ASSEMBLY, + .id = "%ResourceID%" }, .assembly = { .name = "%RegisteredFilename%", .data = %ResourceName%_data, .size = %Len% }, diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template index abd5bcf8801e03..2b2538733f9f8f 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template @@ -1,5 +1,6 @@ const MonoBundledDataResource %ResourceName% = { - .resource = { .type = MONO_BUNDLED_DATA }, + .resource = { .type = MONO_BUNDLED_DATA, + .id = "%ResourceID%" }, .data = { .name = "%RegisteredFilename%", .data = %ResourceName%_data, .size = %Len% }, diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template index 725208a2d84981..19ad0eb32bc07c 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template @@ -1,5 +1,6 @@ const MonoBundledSatelliteAssemblyResource %ResourceName% = { - .resource = { .type = MONO_BUNDLED_SATELLITE_ASSEMBLY }, + .resource = { .type = MONO_BUNDLED_SATELLITE_ASSEMBLY, + .id = "%ResourceID%" }, .satellite_assembly = { .name = "%RegisteredFilename%", .data = %ResourceName%_data, .size = %Len% }, diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h index 0ade0dc19d7d71..b7617ade274b29 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h @@ -18,6 +18,7 @@ typedef enum { typedef struct _MonoBundledResource { MonoBundledResourceType type; + const char *id; void (*free_bundled_resource_func)(void *); } MonoBundledResource; From 7f4258a57ab390baccf25cffbdd059b8360fb09d Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 5 May 2023 13:59:25 -0400 Subject: [PATCH 042/110] Include culture information for satellite resources --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 11 ++++------- .../mono-bundled-satellite-assembly.template | 1 + 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 72bc3d959450e2..bf31a37469385c 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -83,15 +83,11 @@ public override bool Execute() var outputFile = registeredFile.GetMetadata("DestinationFile"); var registeredFilename = group.Key; var resourceName = ToSafeSymbolName(outputFile); + string culture = registeredFile.GetMetadata("Culture"); string? resourceSymbolName = null; -<<<<<<< HEAD if (File.Exists(registeredFile.GetMetadata("SymbolFile"))) resourceSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("SymbolFile")); -======= - if (File.Exists(registeredFile.GetMetadata("Symfile"))) - resourceSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("Symfile")); ->>>>>>> 5a56c0e2b21 (Differentiate resource name and symbol) - return (registeredFilename, resourceName, resourceSymbolName); + return (registeredFilename, resourceName, culture, resourceSymbolName); }).ToList(); Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); @@ -192,7 +188,7 @@ private static byte[] InitLookupTable() private static Dictionary symbolDataLen = new(); - private static void GenerateBundledResourcePreallocationAndRegistration(string bundleRegistrationFunctionName, ICollection<(string registeredFilename, string resourceName, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) + private static void GenerateBundledResourcePreallocationAndRegistration(string bundleRegistrationFunctionName, ICollection<(string registeredFilename, string resourceName, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) { StringBuilder preallocatedSource = new(); @@ -231,6 +227,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b } case "MONO_BUNDLED_SATELLITE_ASSEMBLY": { preloadedStruct = satelliteAssemblyTemplate; + preloadedStruct.Replace("%Culture%", tuple.culture); resourceId = $"{tuple.culture}/{tuple.registeredFilename}"; preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); satelliteAssembliesCount += 1; diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template index 19ad0eb32bc07c..f4a12e1c6e0eac 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template @@ -2,6 +2,7 @@ const MonoBundledSatelliteAssemblyResource %ResourceName% = { .resource = { .type = MONO_BUNDLED_SATELLITE_ASSEMBLY, .id = "%ResourceID%" }, .satellite_assembly = { .name = "%RegisteredFilename%", + .culture = "%Culture%", .data = %ResourceName%_data, .size = %Len% }, %MonoBundledSymbolData%}; \ No newline at end of file From a632a739131a4664f9757473dd26f710911fd9bb Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 5 May 2023 16:41:03 -0400 Subject: [PATCH 043/110] Generate preallocation and registration source file iff BundleFile provided --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index bf31a37469385c..030cdd8b9c8915 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -33,7 +33,7 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab /// The filename for the generated source file that registers /// the bundled resources. /// - public string BundleFile { get; set; } = "mono-bundled-source.c"; + public string? BundleFile { get; set; } /// /// The filename for the generated header file that declares @@ -136,17 +136,19 @@ public override bool Execute() }); } - // Generate header containing MonoBundled*Resource typedefs - File.WriteAllText(Path.Combine(OutputDirectory, BundleHeader), Utils.GetEmbeddedResource("mono-bundled-source.h")); + if (!string.IsNullOrEmpty(BundleFile)) { + // Generate header containing MonoBundled*Resource typedefs + File.WriteAllText(Path.Combine(OutputDirectory, BundleHeader), Utils.GetEmbeddedResource("mono-bundled-source.h")); - // Generate source file to preallocate resources and register bundled resources - Emit(Path.Combine(OutputDirectory, BundleFile), (inputStream) => - { - using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); - GenerateBundledResourcePreallocationAndRegistration(BundleRegistrationFunctionName, files, outputUtf8Writer); - }); + // Generate source file to preallocate resources and register bundled resources + Emit(Path.Combine(OutputDirectory, BundleFile), (inputStream) => + { + using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); + GenerateBundledResourcePreallocationAndRegistration(BundleRegistrationFunctionName, files, outputUtf8Writer); + }); + } - return true; + return !Log.HasLoggedErrors; } public void Cancel() From 429fe75fae8b25f948bdab942633d8546493ea46 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 5 May 2023 16:41:35 -0400 Subject: [PATCH 044/110] Generate output itemgroup detailing resource data --- src/mono/msbuild/common/LibraryBuilder.targets | 5 +++-- src/tasks/EmitBundleTask/EmitBundleBase.cs | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index 8b912954062402..b400acde97a8bd 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -79,8 +79,9 @@ FilesToBundle="@(_ResourcesToBundleWithHashes)" CombinedResourceSource="$(BundleResourceFile)" BundleFile="$(BundleSourceFile)" - OutputDirectory="$(BundleOutputDirectory)" - /> + OutputDirectory="$(BundleOutputDirectory)"> + + \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 030cdd8b9c8915..6c6abde3a05c72 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -53,6 +53,16 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab /// public string OutputDirectory {get; set; } = default!; + /// + /// Resources that were bundled + /// + /// Successful bundling will set the following metadata on the items: + /// - DataSymbol + /// - LenSymbol + /// + [Output] + public ITaskItem[] BundledResources { get; set; } = default!; + public override bool Execute() { // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore @@ -92,6 +102,7 @@ public override bool Execute() Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); + List bundledResources = new(remainingDestinationFilesToBundle.Length); // Generate source file(s) containing each resource's byte data and size if (remainingDestinationFilesToBundle.Length > 0) { @@ -133,8 +144,13 @@ public override bool Execute() { state.Stop(); } + contentSourceFile.SetMetadata("DataSymbol", $"{symbolName}_data"); + contentSourceFile.SetMetadata("LenSymbol", $"{symbolName}_len"); + contentSourceFile.SetMetadata("LenSymbolValue", symbolDataLen[symbolName]); + bundledResources.Add(contentSourceFile); }); } + BundledResources = bundledResources.ToArray(); if (!string.IsNullOrEmpty(BundleFile)) { // Generate header containing MonoBundled*Resource typedefs From 5aeaef4ab9272a7d81127304a643bf55ea261839 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Sun, 7 May 2023 16:28:36 -0400 Subject: [PATCH 045/110] Leverage new bundling api from old bundling api and deprecate --- src/mono/mono/metadata/assembly.c | 86 ++++++++++--------- .../metadata/details/assembly-functions.h | 2 +- .../details/mono-private-unstable-functions.h | 2 +- 3 files changed, 46 insertions(+), 44 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 2197d23c151513..e655f4073392ca 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -81,10 +81,6 @@ mono_assemblies_unlock (void) mono_os_mutex_unlock (&assemblies_mutex); } -/* If defined, points to the bundled assembly information */ -static const MonoBundledAssembly **bundles; -static const MonoBundledSatelliteAssembly **satellite_bundles; - /* Class lazy loading functions */ static GENERATE_TRY_GET_CLASS_WITH_CACHE (debuggable_attribute, "System.Diagnostics", "DebuggableAttribute") @@ -710,8 +706,8 @@ mono_assembly_get_assemblyref (MonoImage *image, int index, MonoAssemblyName *an static MonoAssembly * search_bundle_for_assembly (MonoAssemblyLoadContext *alc, MonoAssemblyName *aname) { - if (bundles == NULL && !mono_bundled_resources_contains_assemblies () && - satellite_bundles == NULL && !mono_bundled_resources_contains_satellite_assemblies ()) + if (!mono_bundled_resources_contains_assemblies () && + !mono_bundled_resources_contains_satellite_assemblies ()) return NULL; MonoImageOpenStatus status; @@ -794,7 +790,7 @@ netcore_load_reference (MonoAssemblyName *aname, MonoAssemblyLoadContext *alc, M } } - if ((bundles != NULL || mono_bundled_resources_contains_assemblies ()) && !is_satellite) { + if (mono_bundled_resources_contains_assemblies () && !is_satellite) { reference = search_bundle_for_assembly (mono_alc_get_default (), aname); if (reference) { mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "Assembly found in the bundle: '%s'.", aname->name); @@ -802,7 +798,7 @@ netcore_load_reference (MonoAssemblyName *aname, MonoAssemblyLoadContext *alc, M } } - if ((satellite_bundles != NULL || mono_bundled_resources_contains_satellite_assemblies ()) && is_satellite) { + if (mono_bundled_resources_contains_satellite_assemblies () && is_satellite) { // Satellite assembly byname requests should be loaded in the same ALC as their parent assembly size_t name_len = strlen (aname->name); char *parent_name = NULL; @@ -1473,45 +1469,25 @@ bundled_assembly_match (const char *bundled_name, const char *name) static MonoImage * open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status) { - if (!bundles && !mono_bundled_resources_contains_assemblies ()) - return NULL; - MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (filename); - if (assembly) - return mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, filename, NULL); - - for (int i = 0; bundles [i]; ++i) { - if (bundled_assembly_match (bundles[i]->name, filename)) { - // Since bundled images don't exist on disk, don't give them a legit filename - return mono_image_open_from_data_internal (alc, (char*)bundles [i]->data, bundles [i]->size, FALSE, status, FALSE, filename, NULL); - } - } + if (!assembly) + return NULL; - return NULL; + return mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, filename, NULL); } static MonoImage * open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status, const char *culture) { - if (!satellite_bundles && !mono_bundled_resources_contains_satellite_assemblies ()) - return NULL; - - MonoImage *image = NULL; char *bundle_name = g_strconcat (culture, "/", filename, (const char *)NULL); - MonoBundledSatelliteAssemblyResource *satellite_assembly = mono_bundled_resources_get_satellite_assembly_resource (filename); - if (satellite_assembly) - image = mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, bundle_name, NULL); + MonoBundledSatelliteAssemblyResource *satellite_assembly = mono_bundled_resources_get_satellite_assembly_resource (bundle_name); + g_free (bundle_name); + if (!satellite_assembly) + return NULL; - for (int i = 0; !image && satellite_bundles [i]; ++i) { - if (bundled_assembly_match (satellite_bundles[i]->name, filename) && strcmp (satellite_bundles [i]->culture, culture) == 0) { - // Since bundled images don't exist on disk, don't give them a legit filename - image = mono_image_open_from_data_internal (alc, (char *)satellite_bundles [i]->data, satellite_bundles [i]->size, FALSE, status, FALSE, bundle_name, NULL); - } - } + return mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, bundle_name, NULL); - g_free (bundle_name); - return image; } /** @@ -1520,7 +1496,7 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, * \param status return status code * * This routine tries to open the assembly specified by \p filename from the - * defined bundles, if found, returns the MonoImage for it, if not found + * bundles hashtable in bundled-resources.c, if found, returns the MonoImage for it, if not found * returns NULL */ MonoImage * @@ -1616,7 +1592,7 @@ mono_assembly_request_open (const char *filename, const MonoAssemblyOpenRequest // If VM built with mkbundle loaded_from_bundle = FALSE; - if (bundles != NULL || mono_bundled_resources_contains_assemblies ()) { + if (mono_bundled_resources_contains_assemblies ()) { /* We don't know the culture of the filename we're loading here, so this call is not culture aware. */ image = mono_assembly_open_from_bundle (load_req.alc, fname, status, NULL); loaded_from_bundle = image != NULL; @@ -3163,10 +3139,26 @@ mono_assembly_get_name_internal (MonoAssembly *assembly) /** * mono_register_bundled_assemblies: */ +MONO_DEPRECATED void mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) { - bundles = assemblies; + for (int i = 0; assemblies [i]; ++i) { + const char *id = assemblies [i]->name; + MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (id); + if (!assembly_resource) { + assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); + assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; + assembly_resource->resource.id = id; + } else { + g_assert (assembly_resource->resource.type == MONO_BUNDLED_ASSEMBLY); + g_assert (!strcmp(assembly_resource->resource.id, id)); + // Ensure the MonoBundledAssemblyData has not been initialized + g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); + } + assembly_resource->assembly = *(MonoBundledAssemblyData *)assemblies [i]; + mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); + } } /** @@ -3188,10 +3180,20 @@ mono_create_new_bundled_satellite_assembly (const char *name, const char *cultur /** * mono_register_bundled_satellite_assemblies: */ +MONO_DEPRECATED void -mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly **assemblies) -{ - satellite_bundles = assemblies; +mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly **satellite_assemblies) +{ + for (int i = 0; satellite_assemblies [i]; ++i) { + const MonoBundledSatelliteAssembly *satellite_assembly = satellite_assemblies [i]; + char *id = g_strconcat (satellite_assembly->culture, "/", satellite_assembly->name, (const char*)NULL); + MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); + g_assert (!satellite_assembly_resource); + satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; + satellite_assembly_resource->resource.id = id; + satellite_assembly_resource->satellite_assembly = *satellite_assembly; + mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); + } } /** diff --git a/src/native/public/mono/metadata/details/assembly-functions.h b/src/native/public/mono/metadata/details/assembly-functions.h index df752b66956193..cccc4155ab15f7 100644 --- a/src/native/public/mono/metadata/details/assembly-functions.h +++ b/src/native/public/mono/metadata/details/assembly-functions.h @@ -62,7 +62,7 @@ MONO_API_FUNCTION(uint16_t, mono_assembly_name_get_version, (MonoAssemblyName *a MONO_API_FUNCTION(mono_byte*, mono_assembly_name_get_pubkeytoken, (MonoAssemblyName *aname)) MONO_API_FUNCTION(MONO_RT_EXTERNAL_ONLY void, mono_assembly_name_free, (MonoAssemblyName *aname)) -MONO_API_FUNCTION(void, mono_register_bundled_assemblies, (const MonoBundledAssembly **assemblies)) +MONO_API_FUNCTION(void, mono_register_bundled_assemblies, (const MonoBundledAssembly **assemblies) /* MONO_DEPRECATED */) MONO_API_FUNCTION(void, mono_register_symfile_for_assembly, (const char* assembly_name, const mono_byte *raw_contents, int size)) MONO_API_FUNCTION(const mono_byte *, mono_get_symfile_bytes_from_bundle, (const char* assembly_name, int *size)) diff --git a/src/native/public/mono/metadata/details/mono-private-unstable-functions.h b/src/native/public/mono/metadata/details/mono-private-unstable-functions.h index 0eb931ce1bb7fe..e7e4e31f402290 100644 --- a/src/native/public/mono/metadata/details/mono-private-unstable-functions.h +++ b/src/native/public/mono/metadata/details/mono-private-unstable-functions.h @@ -23,7 +23,7 @@ MONO_API_FUNCTION(MONO_RT_EXTERNAL_ONLY void, mono_install_assembly_preload_hook // This can point at NULL before the default ALC is initialized MONO_API_FUNCTION(MONO_RT_EXTERNAL_ONLY MonoAssemblyLoadContextGCHandle, mono_alc_get_default_gchandle, (void)) -MONO_API_FUNCTION(void, mono_register_bundled_satellite_assemblies, (const MonoBundledSatelliteAssembly **assemblies)) +MONO_API_FUNCTION(void, mono_register_bundled_satellite_assemblies, (const MonoBundledSatelliteAssembly **assemblies) /* MONO_DEPRECATED */) MONO_API_FUNCTION(MonoBundledSatelliteAssembly *, mono_create_new_bundled_satellite_assembly, (const char *name, const char *culture, const unsigned char *data, unsigned int size)) From ccd783b2932a9cb65ced14195b9e700868a76f90 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 9 May 2023 23:57:59 -0400 Subject: [PATCH 046/110] [mono][debug] Migrate symbol data registration to new bundling api --- src/mono/mono/metadata/mono-debug.c | 76 ++++++++--------------------- 1 file changed, 19 insertions(+), 57 deletions(-) diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 1c7a256ef15b06..ea42c14180d9b4 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1067,60 +1067,31 @@ mono_is_debugger_attached (void) return is_attached; } -/* - * Bundles - */ - -typedef struct _BundledSymfile BundledSymfile; - -struct _BundledSymfile { - BundledSymfile *next; - const char *aname; - const mono_byte *raw_contents; - int size; -}; - -static BundledSymfile *bundled_symfiles = NULL; - /** * mono_register_symfile_for_assembly: */ void mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte *raw_contents, int size) { - BundledSymfile *bsymfile; - - bsymfile = g_new0 (BundledSymfile, 1); - bsymfile->aname = assembly_name; - bsymfile->raw_contents = raw_contents; - bsymfile->size = size; - bsymfile->next = bundled_symfiles; - bundled_symfiles = bsymfile; -} - -static gboolean -bsymfile_match (BundledSymfile *bsymfile, const char *assembly_name) -{ - if (!strcmp (bsymfile->aname, assembly_name)) - return TRUE; -#ifdef ENABLE_WEBCIL - const char *p = strstr (assembly_name, ".webcil"); - /* if assembly_name ends with .webcil, check if aname matches, with a .dll extension instead */ - if (p && *(p + strlen(".webcil")) == 0) { - size_t n = p - assembly_name; - if (!strncmp (bsymfile->aname, assembly_name, n) - && !strcmp (bsymfile->aname + n, ".dll")) - return TRUE; + MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (assembly_name); + if (!assembly_resource) { + assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); + assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; + assembly_resource->resource.id = assembly_name; + } else { + g_assert (assembly_resource->resource.type == MONO_BUNDLED_ASSEMBLY); + g_assert (!strcmp(assembly_resource->resource.id, assembly_name)); + // Ensure the MonoBundledSymbolData has not been initialized + g_assert (!assembly_resource->symbol_data.data && assembly_resource->symbol_data.size == 0); } -#endif - return FALSE; + assembly_resource->symbol_data.data = (uint8_t *)raw_contents; + assembly_resource->symbol_data.size = (uint32_t)size; + mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); } static MonoDebugHandle * open_symfile_from_bundle (MonoImage *image) { - BundledSymfile *bsymfile; - MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (image->module_name); if (assembly && assembly->symbol_data.data) return mono_debug_open_image (image, assembly->symbol_data.data, assembly->symbol_data.size); @@ -1139,27 +1110,18 @@ open_symfile_from_bundle (MonoImage *image) return mono_debug_open_image (image, assembly->symfile.data, assembly->symfile.size); #endif - for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { - if (!bsymfile_match (bsymfile, image->module_name)) - continue; - - return mono_debug_open_image (image, bsymfile->raw_contents, bsymfile->size); - } - return NULL; } const mono_byte * mono_get_symfile_bytes_from_bundle (const char *assembly_name, int *size) { - BundledSymfile *bsymfile; - for (bsymfile = bundled_symfiles; bsymfile; bsymfile = bsymfile->next) { - if (!bsymfile_match (bsymfile, assembly_name)) - continue; - *size = bsymfile->size; - return bsymfile->raw_contents; - } - return NULL; + MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (assembly_name); + if (!assembly) + return NULL; + + *size = assembly->symbol_data.size; + return assembly->symbol_data.data; } void From 3cf95fc14047bfb99a54c3608e4c343ebf4387e0 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 9 May 2023 15:52:47 -0400 Subject: [PATCH 047/110] Remove symbol data from MonoBundleSatelliteAssemblyResource --- .../metadata/bundled-resources-internals.h | 1 - src/tasks/EmitBundleTask/EmitBundleBase.cs | 20 +++++++++---------- .../mono-bundled-satellite-assembly.template | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index fdb70a75ae0dbf..5fc5a50daa19d6 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -51,7 +51,6 @@ typedef struct _MonoBundledAssemblyResource { typedef struct _MonoBundledSatelliteAssemblyResource { MonoBundledResource resource; MonoBundledSatelliteAssembly satellite_assembly; - MonoBundledSymbolData symbol_data; } MonoBundledSatelliteAssemblyResource; void diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 6c6abde3a05c72..818ee35a5f7966 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -239,6 +239,15 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b switch (GetFileType(tuple.registeredFilename)) { case "MONO_BUNDLED_ASSEMBLY": { preloadedStruct = assemblyTemplate; + // Add associated symfile information to MonoBundledAssemblyResource structs + string preloadedSymbolData = ""; + if (!string.IsNullOrEmpty(tuple.resourceSymbolName)) + { + preloadedSymbolData = Utils.GetEmbeddedResource("mono-bundled-symbol.template") + .Replace("%ResourceSymbolName%", tuple.resourceSymbolName) + .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName].ToString()); + } + preloadedStruct = preloadedStruct.Replace("%MonoBundledSymbolData%", preloadedSymbolData); preallocatedAssemblies.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); assembliesCount += 1; break; @@ -260,19 +269,10 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b } } - // Add associated symfile information to MonoBundledAssemblyResource/MonoBundleSatelliteAssemblyResource structs - string preloadedSymfile = ""; - if (!string.IsNullOrEmpty(tuple.resourceSymbolName)) - { - preloadedSymfile = Utils.GetEmbeddedResource("mono-bundled-symbol.template") - .Replace("%ResourceSymbolName%", tuple.resourceSymbolName) - .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName].ToString()); - } preallocatedSource.AppendLine(preloadedStruct.Replace("%ResourceName%", tuple.resourceName) .Replace("%ResourceID%", resourceId) .Replace("%RegisteredFilename%", tuple.registeredFilename) - .Replace("%Len%", symbolDataLen[tuple.resourceName].ToString()) - .Replace("%MonoBundledSymbolData%", preloadedSymfile)); + .Replace("%Len%", symbolDataLen[tuple.resourceName].ToString())); } var addPreallocatedResources = new StringBuilder(); diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template index f4a12e1c6e0eac..a239efc7c2a300 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template @@ -5,4 +5,4 @@ const MonoBundledSatelliteAssemblyResource %ResourceName% = { .culture = "%Culture%", .data = %ResourceName%_data, .size = %Len% }, -%MonoBundledSymbolData%}; \ No newline at end of file +}; \ No newline at end of file From 5d265dbfc1f85e5edfdef07da95365f63532364a Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 9 May 2023 15:56:47 -0400 Subject: [PATCH 048/110] Directly determine resource type --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 40 +++++++--------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 818ee35a5f7966..4a7a18098e3972 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -236,8 +236,16 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b // Generate Preloaded MonoBundled*Resource structs string preloadedStruct; - switch (GetFileType(tuple.registeredFilename)) { - case "MONO_BUNDLED_ASSEMBLY": { + if (!string.IsNullOrEmpty(tuple.culture)) + { + preloadedStruct = satelliteAssemblyTemplate; + preloadedStruct.Replace("%Culture%", tuple.culture); + resourceId = $"{tuple.culture}/{tuple.registeredFilename}"; + preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); + satelliteAssembliesCount += 1; + } + else if (tuple.registeredFilename.EndsWith(".dll")) + { preloadedStruct = assemblyTemplate; // Add associated symfile information to MonoBundledAssemblyResource structs string preloadedSymbolData = ""; @@ -250,23 +258,12 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b preloadedStruct = preloadedStruct.Replace("%MonoBundledSymbolData%", preloadedSymbolData); preallocatedAssemblies.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); assembliesCount += 1; - break; } - case "MONO_BUNDLED_SATELLITE_ASSEMBLY": { - preloadedStruct = satelliteAssemblyTemplate; - preloadedStruct.Replace("%Culture%", tuple.culture); - resourceId = $"{tuple.culture}/{tuple.registeredFilename}"; - preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); - satelliteAssembliesCount += 1; - break; - } - case "MONO_BUNDLED_DATA": - default: { + else + { preloadedStruct = symbolDataTemplate; preallocatedData.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); dataCount += 1; - break; - } } preallocatedSource.AppendLine(preloadedStruct.Replace("%ResourceName%", tuple.resourceName) @@ -358,19 +355,6 @@ private static string ToSafeSymbolName(string destinationFileName) return sb.ToString(); } - private static string GetFileType(string destinationFileName) - { - if (destinationFileName.EndsWith(".resources.dll")) - { - return "MONO_BUNDLED_SATELLITE_ASSEMBLY"; - } - if (destinationFileName.EndsWith(".dll")) - { - return "MONO_BUNDLED_ASSEMBLY"; - } - return "MONO_BUNDLED_DATA"; - } - // Equivalent to "isalnum" private static bool IsAlphanumeric(char c) => c is (>= 'a' and <= 'z') From 3206aaba68050a2b746f629ca0c30273ce4cfb78 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 9 May 2023 23:06:39 -0400 Subject: [PATCH 049/110] Update bundled header and switch to non internal types Update bundled header to declare resource symbols and registration function Switch types from mono internal types to C99 compliant as bundling apis are partially exposed to public --- src/mono/mono/metadata/assembly.c | 2 +- .../metadata/bundled-resources-internals.h | 27 +++-- src/tasks/EmitBundleTask/EmitBundleBase.cs | 51 +++++---- ...ce-preallocation-and-registration.template | 60 ++++++++++- .../Templates/mono-bundled-source.h | 101 +----------------- 5 files changed, 115 insertions(+), 126 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index e655f4073392ca..61cba94f5e7372 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -3191,7 +3191,7 @@ mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly * g_assert (!satellite_assembly_resource); satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; satellite_assembly_resource->resource.id = id; - satellite_assembly_resource->satellite_assembly = *satellite_assembly; + satellite_assembly_resource->satellite_assembly = *(MonoBundledSatelliteAssemblyData *)satellite_assembly; mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); } } diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 5fc5a50daa19d6..5a45b0dfe8f0b3 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -27,9 +27,9 @@ typedef struct _MonoBundledResource { } MonoBundledResource; typedef struct _MonoBundledData { - char *name; - const unsigned char *data; - unsigned int size; + const char *name; + const uint8_t *data; + uint32_t size; } MonoBundledData; typedef struct _MonoBundledDataResource { @@ -38,19 +38,32 @@ typedef struct _MonoBundledDataResource { } MonoBundledDataResource; typedef struct _MonoBundledSymbolData { - const unsigned char *data; - unsigned int size; + const uint8_t *data; + uint32_t size; } MonoBundledSymbolData; +typedef struct _MonoBundledAssemblyData { + const char *name; + const uint8_t *data; + uint32_t size; +} MonoBundledAssemblyData; + typedef struct _MonoBundledAssemblyResource { MonoBundledResource resource; - MonoBundledAssembly assembly; + MonoBundledAssemblyData assembly; MonoBundledSymbolData symbol_data; } MonoBundledAssemblyResource; +typedef struct _MonoBundledSatelliteAssemblyData { + const char *name; + const char *culture; + const uint8_t *data; + uint32_t size; +} MonoBundledSatelliteAssemblyData; + typedef struct _MonoBundledSatelliteAssemblyResource { MonoBundledResource resource; - MonoBundledSatelliteAssembly satellite_assembly; + MonoBundledSatelliteAssemblyData satellite_assembly; } MonoBundledSatelliteAssemblyResource; void diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 4a7a18098e3972..e5437d271b12f9 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -102,7 +102,10 @@ public override bool Execute() Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); + HashSet resourceSourcesGenerated = new(); List bundledResources = new(remainingDestinationFilesToBundle.Length); + var resourceSymbols = new StringBuilder(); + string registrationFunction = ""; // Generate source file(s) containing each resource's byte data and size if (remainingDestinationFilesToBundle.Length > 0) { @@ -124,6 +127,12 @@ public override bool Execute() if (!string.IsNullOrEmpty(CombinedResourceSource)) outputFile = Path.Combine(OutputDirectory, CombinedResourceSource); var inputFile = contentSourceFile.ItemSpec; + bool shouldAddHeader = false; + lock (resourceSourcesGenerated) + { + shouldAddHeader = !resourceSourcesGenerated.Contains(outputFile); + } + if (verbose) { var registeredName = contentSourceFile.GetMetadata("RegisteredName"); @@ -139,31 +148,42 @@ public override bool Execute() var symbolName = ToSafeSymbolName(group.Key); if (!Emit(outputFile, (codeStream) => { using var inputStream = File.OpenRead(inputFile); - BundleFileToCSource(symbolName, inputStream, codeStream); + BundleFileToCSource(symbolName, shouldAddHeader, inputStream, codeStream); })) { state.Stop(); } + lock (resourceSourcesGenerated) + { + resourceSourcesGenerated.Add(outputFile); + } + resourceSymbols.AppendLine($"extern uint8_t {symbolName}_data[];"); + resourceSymbols.AppendLine($"extern const uint32_t {symbolName}_len;"); + contentSourceFile.SetMetadata("DataSymbol", $"{symbolName}_data"); contentSourceFile.SetMetadata("LenSymbol", $"{symbolName}_len"); - contentSourceFile.SetMetadata("LenSymbolValue", symbolDataLen[symbolName]); + contentSourceFile.SetMetadata("LenSymbolValue", symbolDataLen[symbolName].ToString()); bundledResources.Add(contentSourceFile); }); } BundledResources = bundledResources.ToArray(); if (!string.IsNullOrEmpty(BundleFile)) { - // Generate header containing MonoBundled*Resource typedefs - File.WriteAllText(Path.Combine(OutputDirectory, BundleHeader), Utils.GetEmbeddedResource("mono-bundled-source.h")); + registrationFunction = $"void {BundleRegistrationFunctionName} (void);"; // Generate source file to preallocate resources and register bundled resources Emit(Path.Combine(OutputDirectory, BundleFile), (inputStream) => { using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); - GenerateBundledResourcePreallocationAndRegistration(BundleRegistrationFunctionName, files, outputUtf8Writer); + GenerateBundledResourcePreallocationAndRegistration(BundleHeader, BundleRegistrationFunctionName, files, outputUtf8Writer); }); } + // Generate header containing resource data/len symbols (+ registration function) + File.WriteAllText(Path.Combine(OutputDirectory, BundleHeader), Utils.GetEmbeddedResource("mono-bundled-source.h") + .Replace("%ResourceSymbols%", resourceSymbols.ToString()) + .Replace("%RegistrationFunction%", registrationFunction)); + return !Log.HasLoggedErrors; } @@ -206,7 +226,7 @@ private static byte[] InitLookupTable() private static Dictionary symbolDataLen = new(); - private static void GenerateBundledResourcePreallocationAndRegistration(string bundleRegistrationFunctionName, ICollection<(string registeredFilename, string resourceName, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) + private static void GenerateBundledResourcePreallocationAndRegistration(string bundleHeader, string bundleRegistrationFunctionName, ICollection<(string registeredFilename, string resourceName, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) { StringBuilder preallocatedSource = new(); @@ -225,15 +245,6 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b { string resourceId = tuple.registeredFilename; - // extern symbols - StringBuilder preallocatedResourceData = new(); - preallocatedResourceData.AppendLine($"extern const unsigned char {tuple.resourceName}_data[];"); - if (!string.IsNullOrEmpty(tuple.resourceSymbolName)) - { - preallocatedResourceData.AppendLine($"extern const unsigned char {tuple.resourceSymbolName}_data[];"); - } - preallocatedSource.AppendLine(preallocatedResourceData.ToString()); - // Generate Preloaded MonoBundled*Resource structs string preloadedStruct; if (!string.IsNullOrEmpty(tuple.culture)) @@ -290,13 +301,14 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b } outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-resource-preallocation-and-registration.template") + .Replace("%BundleHeader%", bundleHeader) .Replace("%PreallocatedStructs%", preallocatedSource.ToString()) .Replace("%PreallocatedResources%", preallocatedResources.ToString()) .Replace("%BundleRegistrationFunctionName%", bundleRegistrationFunctionName) .Replace("%AddPreallocatedResources%", addPreallocatedResources.ToString())); } - private static void BundleFileToCSource(string symbolName, FileStream inputStream, Stream outputStream) + private static void BundleFileToCSource(string symbolName, bool shouldAddHeader, FileStream inputStream, Stream outputStream) { // Emits a C source file in the same format as "xxd --include". Example: // @@ -312,7 +324,10 @@ private static void BundleFileToCSource(string symbolName, FileStream inputStrea using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); - outputUtf8Writer.Write($"unsigned char {symbolName}_data[] = {{"); + if (shouldAddHeader) + outputUtf8Writer.WriteLine("#include "); + + outputUtf8Writer.Write($"uint8_t {symbolName}_data[] = {{"); outputUtf8Writer.Flush(); while ((bytesRead = inputStream.Read(buf, 0, buf.Length)) > 0) { @@ -331,7 +346,7 @@ private static void BundleFileToCSource(string symbolName, FileStream inputStrea } outputUtf8Writer.WriteLine("0\n};"); - outputUtf8Writer.WriteLine($"unsigned int {symbolName}_len = {generatedArrayLength};"); + outputUtf8Writer.WriteLine($"const uint32_t {symbolName}_len = {generatedArrayLength};"); outputUtf8Writer.Flush(); outputStream.Flush(); diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template index 004c5ddac5c52e..e12db6c061b2e7 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template @@ -1,7 +1,65 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#include "mono-bundled-source.h" +#include + +#include "%BundleHeader%" + +typedef enum { + MONO_BUNDLED_DATA, + MONO_BUNDLED_ASSEMBLY, + MONO_BUNDLED_SATELLITE_ASSEMBLY, + MONO_BUNDLED_RESOURCE_COUNT, +} MonoBundledResourceType; + +typedef struct _MonoBundledResource { + MonoBundledResourceType type; + const char *id; + void (*free_bundled_resource_func)(void *); +} MonoBundledResource; + +typedef struct _MonoBundledData { + const char *name; + const uint8_t *data; + uint32_t size; +} MonoBundledData; + +typedef struct _MonoBundledDataResource { + MonoBundledResource resource; + MonoBundledData data; +} MonoBundledDataResource; + +typedef struct _MonoBundledSymbolData { + const uint8_t *data; + uint32_t size; +} MonoBundledSymbolData; + +typedef struct _MonoBundledAssemblyData { + const char *name; + const uint8_t *data; + uint32_t size; +} MonoBundledAssemblyData; + +typedef struct _MonoBundledAssemblyResource { + MonoBundledResource resource; + MonoBundledAssemblyData assembly; + MonoBundledSymbolData symbol_data; +} MonoBundledAssemblyResource; + +typedef struct _MonoBundledSatelliteAssemblyData { + const char *name; + const char *culture; + const uint8_t *data; + uint32_t size; +} MonoBundledSatelliteAssemblyData; + +typedef struct _MonoBundledSatelliteAssemblyResource { + MonoBundledResource resource; + MonoBundledSatelliteAssemblyData satellite_assembly; +} MonoBundledSatelliteAssemblyResource; + +void +mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); %PreallocatedStructs% diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h index b7617ade274b29..ba8a795dfbe3bd 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h @@ -4,105 +4,8 @@ #ifndef __MONO_BUNDLED_SOURCE_H__ #define __MONO_BUNDLED_SOURCE_H__ -#include -#include +%ResourceSymbols% -#include - -typedef enum { - MONO_BUNDLED_DATA, - MONO_BUNDLED_ASSEMBLY, - MONO_BUNDLED_SATELLITE_ASSEMBLY, - MONO_BUNDLED_RESOURCE_COUNT, -} MonoBundledResourceType; - -typedef struct _MonoBundledResource { - MonoBundledResourceType type; - const char *id; - void (*free_bundled_resource_func)(void *); -} MonoBundledResource; - -typedef struct _MonoBundledData { - char *name; - const unsigned char *data; - unsigned int size; -} MonoBundledData; - -typedef struct _MonoBundledDataResource { - MonoBundledResource resource; - MonoBundledData data; -} MonoBundledDataResource; - -typedef struct _MonoBundledSymbolData { - const unsigned char *data; - unsigned int size; -} MonoBundledSymbolData; - -typedef struct _MonoBundledAssemblyResource { - MonoBundledResource resource; - MonoBundledAssembly assembly; - MonoBundledSymbolData symbol_data; -} MonoBundledAssemblyResource; - -typedef struct _MonoBundledSatelliteAssembly { - const char *name; - const char *culture; - const unsigned char *data; - unsigned int size; -} MonoBundledSatelliteAssembly; - -typedef struct _MonoBundledSatelliteAssemblyResource { - MonoBundledResource resource; - MonoBundledSatelliteAssembly satellite_assembly; - MonoBundledSymbolData symbol_data; -} MonoBundledSatelliteAssemblyResource; - -void -mono_bundled_resources_free (void); - -void -mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); - -MonoBundledResource * -mono_bundled_resources_get (const char *name); - -static inline MonoBundledAssemblyResource * -mono_bundled_resources_get_assembly_resource (const char *name) -{ - MonoBundledAssemblyResource *assembly = - (MonoBundledAssemblyResource*)mono_bundled_resources_get (name); - if (!assembly) - return NULL; - assert (assembly->resource.type == MONO_BUNDLED_ASSEMBLY); - return assembly; -} - -static inline MonoBundledSatelliteAssemblyResource * -mono_bundled_resources_get_satellite_assembly_resource (const char *name) -{ - MonoBundledSatelliteAssemblyResource *satellite_assembly = - (MonoBundledSatelliteAssemblyResource*)mono_bundled_resources_get (name); - if (!satellite_assembly) - return NULL; - assert (satellite_assembly->resource.type == MONO_BUNDLED_SATELLITE_ASSEMBLY); - return satellite_assembly; -} - -static inline MonoBundledDataResource * -mono_bundled_resources_get_data_resource (const char *name) -{ - MonoBundledDataResource *data = - (MonoBundledDataResource*)mono_bundled_resources_get (name); - if (!data) - return NULL; - assert (data->resource.type == MONO_BUNDLED_DATA); - return data; -} - -bool -mono_bundled_resources_contains_assemblies (void); - -bool -mono_bundled_resources_contains_satellite_assemblies (void); +%RegistrationFunction% #endif /* __MONO_BUNDLED_SOURCE_H__ */ \ No newline at end of file From a8c5d88ab7a25f4c31ac24f740f181a537948971 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 9 May 2023 23:11:01 -0400 Subject: [PATCH 050/110] Update Autoinitialization logic to leverage bundled runtimeconfig symbols --- .../msbuild/common/LibraryBuilder.targets | 4 ++ src/tasks/LibraryBuilder/LibraryBuilder.cs | 31 ++++++++++-- src/tasks/LibraryBuilder/Templates/autoinit.c | 47 +++++++++---------- 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index b400acde97a8bd..bf8ba8ce245059 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -21,12 +21,16 @@ <_ExtraLibrarySources Include="$(_AotModuleTablePath)" /> <_ExtraLibrarySources Include="@(BundledSources)" /> <_ExtraLinkerArgs Include="@(_CommonLinkerArgs)" /> + <_BundledRuntimeConfig Include="@(_BundledResources)" Condition="$([System.String]::Copy('%(Identity)').EndsWith('runtimeconfig.bin'))" /> + + public bool BundlesResources { get; set; } + /// + /// An Item containing the bundled runtimeconfig.bin metadata populated by EmitBundle task + /// + public ITaskItem[] BundledRuntimeConfig { get; set; } = default!; + public bool StripDebugSymbols { get; set; } /// @@ -150,6 +155,10 @@ public override bool Execute() if (UsesRuntimeInitCallback && !UsesCustomRuntimeInitCallback) { + if (BundlesResources && BundledRuntimeConfig.Length == 0) + { + throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' is required when bundling and using the default runtime callback."); + } WriteAutoInitializationFromTemplate(); extraSources.AppendLine(" autoinit.c"); } @@ -278,10 +287,26 @@ private static void WriteLinkerScriptFile(string exportsFile, List expor private void WriteAutoInitializationFromTemplate() { - File.WriteAllText(Path.Combine(OutputDirectory, "autoinit.c"), - Utils.GetEmbeddedResource("autoinit.c") + string autoInitialization = Utils.GetEmbeddedResource("autoinit.c") .Replace("%ASSEMBLIES_LOCATION%", !string.IsNullOrEmpty(AssembliesLocation) ? AssembliesLocation : "DOTNET_LIBRARY_ASSEMBLY_PATH") - .Replace("%RUNTIME_IDENTIFIER%", RuntimeIdentifier)); + .Replace("%RUNTIME_IDENTIFIER%", RuntimeIdentifier); + + if (BundlesResources) + { + var bundledRuntimeConfigInfo = BundledRuntimeConfig.First(); + string dataSymbol = bundledRuntimeConfigInfo.GetMetadata("DataSymbol"); + if (string.IsNullOrEmpty(dataSymbol)) + throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataSymbol' metadata."); + string lenSymbol = bundledRuntimeConfigInfo.GetMetadata("LenSymbol"); + if (string.IsNullOrEmpty(lenSymbol)) + throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'LenSymbol' metadata."); + + autoInitialization = autoInitialization + .Replace("%RUNTIME_CONFIG_DATA%", dataSymbol) + .Replace("%RUNTIME_CONFIG_DATA_LEN%", lenSymbol); + } + + File.WriteAllText(Path.Combine(OutputDirectory, "autoinit.c"), autoInitialization); } private void GenerateAssembliesLoader() diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index 8f22058330db5e..1f9269035dfeaf 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -38,33 +38,30 @@ initialize_runtimeconfig (const char *bundle_path) if (!arg) LOG_ERROR ("Out of memory.\n"); - const char *file_name = "runtimeconfig.bin"; - MonoBundledDataResource *runtimeConfig = mono_bundled_resources_get_data_resource (file_name); - - if (runtimeConfig) { - arg->kind = 1; - arg->runtimeconfig.data.data = (const char *)runtimeConfig->data.data; - arg->runtimeconfig.data.data_len = runtimeConfig->data.size; - } else { - size_t str_len = sizeof (char) * (strlen (bundle_path) + strlen (file_name) + 2); // +1 "/", +1 null-terminating char - char *file_path = (char *)malloc (str_len); - if (!file_path) - LOG_ERROR ("Out of memory.\n"); - - int num_char = snprintf (file_path, str_len, "%s/%s", bundle_path, file_name); - if (num_char <= 0 || num_char >= str_len) - LOG_ERROR ("Encoding error while formatting '%s' and '%s' into \"%%s/%%s\".\n", bundle_path, file_name); - - struct stat buffer; - if (stat (file_path, &buffer) != 0) { - free (file_path); - return; - } - - arg->kind = 0; - arg->runtimeconfig.name.path = file_path; +#if defined(BUNDLED_RESOURCES) + arg->kind = 1; + arg->runtimeconfig.data.data = %RUNTIME_CONFIG_DATA%; + arg->runtimeconfig.data.data_len = %RUNTIME_CONFIG_DATA_LEN%; +#else + size_t str_len = sizeof (char) * (strlen (bundle_path) + strlen (file_name) + 2); // +1 "/", +1 null-terminating char + char *file_path = (char *)malloc (str_len); + if (!file_path) + LOG_ERROR ("Out of memory.\n"); + + int num_char = snprintf (file_path, str_len, "%s/%s", bundle_path, file_name); + if (num_char <= 0 || num_char >= str_len) + LOG_ERROR ("Encoding error while formatting '%s' and '%s' into \"%%s/%%s\".\n", bundle_path, file_name); + + struct stat buffer; + if (stat (file_path, &buffer) != 0) { + free (file_path); + return; } + arg->kind = 0; + arg->runtimeconfig.name.path = file_path; +#endif // BUNDLED_RESOURCES + monovm_runtimeconfig_initialize (arg, cleanup_runtime_config, NULL); } From fbe022ea0ce0b9c0b2484f16438cfcde2d37c245 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Wed, 10 May 2023 17:00:49 -0400 Subject: [PATCH 051/110] Integrate header into bundled source --- .../msbuild/common/LibraryBuilder.targets | 1 - src/tasks/EmitBundleTask/EmitBundleBase.cs | 22 +++++-------------- ...ce-preallocation-and-registration.template | 2 +- .../Templates/mono-bundled-source.h | 11 ---------- src/tasks/LibraryBuilder/Templates/autoinit.c | 5 ++++- 5 files changed, 10 insertions(+), 31 deletions(-) delete mode 100644 src/tasks/EmitBundleTask/Templates/mono-bundled-source.h diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index bf8ba8ce245059..63a272328bf3df 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -58,7 +58,6 @@ - diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index e5437d271b12f9..fed2f2bd400c87 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -35,12 +35,6 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab /// public string? BundleFile { get; set; } - /// - /// The filename for the generated header file that declares - /// MonoBundled*Resource struct types. - /// - public string BundleHeader { get; set; } = "mono-bundled-source.h"; - /// /// Filename for the unified source containing all byte data and len /// of the FilesToBundle resources. @@ -105,7 +99,6 @@ public override bool Execute() HashSet resourceSourcesGenerated = new(); List bundledResources = new(remainingDestinationFilesToBundle.Length); var resourceSymbols = new StringBuilder(); - string registrationFunction = ""; // Generate source file(s) containing each resource's byte data and size if (remainingDestinationFilesToBundle.Length > 0) { @@ -159,6 +152,7 @@ public override bool Execute() } resourceSymbols.AppendLine($"extern uint8_t {symbolName}_data[];"); resourceSymbols.AppendLine($"extern const uint32_t {symbolName}_len;"); + resourceSymbols.AppendLine($"#define {symbolName}_len_val {symbolDataLen[symbolName]}"); contentSourceFile.SetMetadata("DataSymbol", $"{symbolName}_data"); contentSourceFile.SetMetadata("LenSymbol", $"{symbolName}_len"); @@ -169,21 +163,15 @@ public override bool Execute() BundledResources = bundledResources.ToArray(); if (!string.IsNullOrEmpty(BundleFile)) { - registrationFunction = $"void {BundleRegistrationFunctionName} (void);"; // Generate source file to preallocate resources and register bundled resources Emit(Path.Combine(OutputDirectory, BundleFile), (inputStream) => { using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); - GenerateBundledResourcePreallocationAndRegistration(BundleHeader, BundleRegistrationFunctionName, files, outputUtf8Writer); + GenerateBundledResourcePreallocationAndRegistration(resourceSymbols.ToString(), BundleRegistrationFunctionName, files, outputUtf8Writer); }); } - // Generate header containing resource data/len symbols (+ registration function) - File.WriteAllText(Path.Combine(OutputDirectory, BundleHeader), Utils.GetEmbeddedResource("mono-bundled-source.h") - .Replace("%ResourceSymbols%", resourceSymbols.ToString()) - .Replace("%RegistrationFunction%", registrationFunction)); - return !Log.HasLoggedErrors; } @@ -226,7 +214,7 @@ private static byte[] InitLookupTable() private static Dictionary symbolDataLen = new(); - private static void GenerateBundledResourcePreallocationAndRegistration(string bundleHeader, string bundleRegistrationFunctionName, ICollection<(string registeredFilename, string resourceName, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) + private static void GenerateBundledResourcePreallocationAndRegistration(string resourceSymbols, string bundleRegistrationFunctionName, ICollection<(string registeredFilename, string resourceName, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) { StringBuilder preallocatedSource = new(); @@ -280,7 +268,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b preallocatedSource.AppendLine(preloadedStruct.Replace("%ResourceName%", tuple.resourceName) .Replace("%ResourceID%", resourceId) .Replace("%RegisteredFilename%", tuple.registeredFilename) - .Replace("%Len%", symbolDataLen[tuple.resourceName].ToString())); + .Replace("%Len%", $"{tuple.resourceName}_len_val")); } var addPreallocatedResources = new StringBuilder(); @@ -301,7 +289,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string b } outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-resource-preallocation-and-registration.template") - .Replace("%BundleHeader%", bundleHeader) + .Replace("%ResourceSymbols%", resourceSymbols) .Replace("%PreallocatedStructs%", preallocatedSource.ToString()) .Replace("%PreallocatedResources%", preallocatedResources.ToString()) .Replace("%BundleRegistrationFunctionName%", bundleRegistrationFunctionName) diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template index e12db6c061b2e7..6c52956d30e37a 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template @@ -3,7 +3,7 @@ #include -#include "%BundleHeader%" +%ResourceSymbols% typedef enum { MONO_BUNDLED_DATA, diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h b/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h deleted file mode 100644 index ba8a795dfbe3bd..00000000000000 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-source.h +++ /dev/null @@ -1,11 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#ifndef __MONO_BUNDLED_SOURCE_H__ -#define __MONO_BUNDLED_SOURCE_H__ - -%ResourceSymbols% - -%RegistrationFunction% - -#endif /* __MONO_BUNDLED_SOURCE_H__ */ \ No newline at end of file diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index 1f9269035dfeaf..3e45524114ff05 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -17,9 +17,12 @@ #include #include "library-builder.h" -#include "mono-bundled-source.h" +#if defined(BUNDLED_RESOURCES) void mono_register_resources_bundle (void); +extern uint8_t %RUNTIME_CONFIG_DATA%[]; +extern const uint32_t %RUNTIME_CONFIG_DATA_LEN%; +#endif static void cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) From 3390e90bd792e5399a580c208d0e5b96446f9114 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Wed, 10 May 2023 20:16:53 -0400 Subject: [PATCH 052/110] Add functions to free allocated MonoBundledResources --- src/mono/mono/metadata/assembly.c | 2 + .../metadata/bundled-resources-internals.h | 3 ++ src/mono/mono/metadata/bundled-resources.c | 38 ++++++++++++++++++- src/mono/mono/metadata/mono-debug.c | 1 + 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 61cba94f5e7372..4b72897c8f034d 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -3150,6 +3150,7 @@ mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; assembly_resource->resource.id = id; + assembly_resource->resource.free_bundled_resource_func = &mono_bundled_resources_free_bundled_resource_func; } else { g_assert (assembly_resource->resource.type == MONO_BUNDLED_ASSEMBLY); g_assert (!strcmp(assembly_resource->resource.id, id)); @@ -3191,6 +3192,7 @@ mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly * g_assert (!satellite_assembly_resource); satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; satellite_assembly_resource->resource.id = id; + satellite_assembly_resource->resource.free_bundled_resource_func = &mono_bundled_resources_free_bundled_resource_func; satellite_assembly_resource->satellite_assembly = *(MonoBundledSatelliteAssemblyData *)satellite_assembly; mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); } diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 5a45b0dfe8f0b3..5f7e235d5fb4b1 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -69,6 +69,9 @@ typedef struct _MonoBundledSatelliteAssemblyResource { void mono_bundled_resources_free (void); +void +mono_bundled_resources_free_bundled_resource_func (void *resource); + void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 8d9c24592d43ed..b490591c3aad38 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -31,6 +31,42 @@ mono_bundled_resources_free (void) bundle_contains_satellite_assemblies = false; } +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_free_bundled_resource_func frees all memory allocated for a +// particular bundled resource. A MonoBundled*Resource should set the underlying +// MonoBundledResource's free_bundled_resource_func to this if it is being dynamically +// allocated (i.e. through old bundling api's). +// +// Through the old bundling api's, allocations occur for the MonoBundled*Resource. +// In addition, MonoBundledSatelliteAssemblyResource's allocate for the id. +// + +void +mono_bundled_resources_free_bundled_resource_func (void *resource) +{ + MonoBundledResource *bundled_resource = (MonoBundledResource *)resource; + + if (bundled_resource->type == MONO_BUNDLED_SATELLITE_ASSEMBLY) + free ((void *)bundled_resource->id); + + free (resource); +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_value_destroy_func frees the memory allocated by the hashtable's +// MonoBundled*Resource by invoking its underlying free_bundled_resource_func when possible. +// + +static void +mono_bundled_resources_value_destroy_func (void *resource) +{ + MonoBundledResource *value = (MonoBundledResource *)resource; + if (value->free_bundled_resource_func) + value->free_bundled_resource_func (resource); +} + //--------------------------------------------------------------------------------------- // // mono_bundled_resources_add handles bundling of many types of resources to circumvent @@ -54,7 +90,7 @@ void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len) { if (!bundled_resources) - bundled_resources = g_hash_table_new (g_str_hash, g_str_equal); + bundled_resources = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, mono_bundled_resources_value_destroy_func); bool assemblyAdded, satelliteAssemblyAdded; diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index ea42c14180d9b4..dd271be3d9a597 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1078,6 +1078,7 @@ mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte * assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; assembly_resource->resource.id = assembly_name; + assembly_resource->resource.free_bundled_resource_func = &mono_bundled_resources_free_bundled_resource_func; } else { g_assert (assembly_resource->resource.type == MONO_BUNDLED_ASSEMBLY); g_assert (!strcmp(assembly_resource->resource.id, assembly_name)); From e16e4673c5f909cadd80d77ad4f56c1d10893ad5 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 11 May 2023 18:20:22 -0400 Subject: [PATCH 053/110] Fixup Leverage new bundling api from old bundling api and deprecate --- src/mono/mono/metadata/assembly.c | 27 ++++++++++--------- .../metadata/details/assembly-functions.h | 2 +- .../details/mono-private-unstable-functions.h | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 4b72897c8f034d..99070d07dda7c4 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1473,7 +1473,7 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M if (!assembly) return NULL; - return mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, filename, NULL); + return mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, assembly->resource.id, NULL); } static MonoImage * @@ -1486,7 +1486,7 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, if (!satellite_assembly) return NULL; - return mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, bundle_name, NULL); + return mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, satellite_assembly->resource.id, NULL); } @@ -3139,25 +3139,24 @@ mono_assembly_get_name_internal (MonoAssembly *assembly) /** * mono_register_bundled_assemblies: */ -MONO_DEPRECATED void mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) { for (int i = 0; assemblies [i]; ++i) { - const char *id = assemblies [i]->name; - MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (id); + const MonoBundledAssembly *assembly = assemblies [i]; + MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (assembly->name); if (!assembly_resource) { assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; - assembly_resource->resource.id = id; - assembly_resource->resource.free_bundled_resource_func = &mono_bundled_resources_free_bundled_resource_func; + assembly_resource->resource.id = assembly->name; + assembly_resource->resource.free_bundled_resource_func = mono_bundled_resources_free_bundled_resource_func; } else { - g_assert (assembly_resource->resource.type == MONO_BUNDLED_ASSEMBLY); - g_assert (!strcmp(assembly_resource->resource.id, id)); // Ensure the MonoBundledAssemblyData has not been initialized g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); } - assembly_resource->assembly = *(MonoBundledAssemblyData *)assemblies [i]; + assembly_resource->assembly.name = assembly->name; + assembly_resource->assembly.data = (const uint8_t *)assembly->data; + assembly_resource->assembly.size = (uint32_t)assembly->size; mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); } } @@ -3181,7 +3180,6 @@ mono_create_new_bundled_satellite_assembly (const char *name, const char *cultur /** * mono_register_bundled_satellite_assemblies: */ -MONO_DEPRECATED void mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly **satellite_assemblies) { @@ -3192,8 +3190,11 @@ mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly * g_assert (!satellite_assembly_resource); satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; satellite_assembly_resource->resource.id = id; - satellite_assembly_resource->resource.free_bundled_resource_func = &mono_bundled_resources_free_bundled_resource_func; - satellite_assembly_resource->satellite_assembly = *(MonoBundledSatelliteAssemblyData *)satellite_assembly; + satellite_assembly_resource->resource.free_bundled_resource_func = mono_bundled_resources_free_bundled_resource_func; + satellite_assembly_resource->satellite_assembly.name = satellite_assembly->name; + satellite_assembly_resource->satellite_assembly.culture = satellite_assembly->culture; + satellite_assembly_resource->satellite_assembly.data = (const uint8_t *)satellite_assembly->data; + satellite_assembly_resource->satellite_assembly.size = (uint32_t)satellite_assembly->size; mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); } } diff --git a/src/native/public/mono/metadata/details/assembly-functions.h b/src/native/public/mono/metadata/details/assembly-functions.h index cccc4155ab15f7..df752b66956193 100644 --- a/src/native/public/mono/metadata/details/assembly-functions.h +++ b/src/native/public/mono/metadata/details/assembly-functions.h @@ -62,7 +62,7 @@ MONO_API_FUNCTION(uint16_t, mono_assembly_name_get_version, (MonoAssemblyName *a MONO_API_FUNCTION(mono_byte*, mono_assembly_name_get_pubkeytoken, (MonoAssemblyName *aname)) MONO_API_FUNCTION(MONO_RT_EXTERNAL_ONLY void, mono_assembly_name_free, (MonoAssemblyName *aname)) -MONO_API_FUNCTION(void, mono_register_bundled_assemblies, (const MonoBundledAssembly **assemblies) /* MONO_DEPRECATED */) +MONO_API_FUNCTION(void, mono_register_bundled_assemblies, (const MonoBundledAssembly **assemblies)) MONO_API_FUNCTION(void, mono_register_symfile_for_assembly, (const char* assembly_name, const mono_byte *raw_contents, int size)) MONO_API_FUNCTION(const mono_byte *, mono_get_symfile_bytes_from_bundle, (const char* assembly_name, int *size)) diff --git a/src/native/public/mono/metadata/details/mono-private-unstable-functions.h b/src/native/public/mono/metadata/details/mono-private-unstable-functions.h index e7e4e31f402290..0eb931ce1bb7fe 100644 --- a/src/native/public/mono/metadata/details/mono-private-unstable-functions.h +++ b/src/native/public/mono/metadata/details/mono-private-unstable-functions.h @@ -23,7 +23,7 @@ MONO_API_FUNCTION(MONO_RT_EXTERNAL_ONLY void, mono_install_assembly_preload_hook // This can point at NULL before the default ALC is initialized MONO_API_FUNCTION(MONO_RT_EXTERNAL_ONLY MonoAssemblyLoadContextGCHandle, mono_alc_get_default_gchandle, (void)) -MONO_API_FUNCTION(void, mono_register_bundled_satellite_assemblies, (const MonoBundledSatelliteAssembly **assemblies) /* MONO_DEPRECATED */) +MONO_API_FUNCTION(void, mono_register_bundled_satellite_assemblies, (const MonoBundledSatelliteAssembly **assemblies)) MONO_API_FUNCTION(MonoBundledSatelliteAssembly *, mono_create_new_bundled_satellite_assembly, (const char *name, const char *culture, const unsigned char *data, unsigned int size)) From 684b66792ea05100267205d17b154b34fa6cdf7e Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 11 May 2023 18:21:29 -0400 Subject: [PATCH 054/110] Fixup Address various fixups --- src/mono/mono/metadata/assembly.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 99070d07dda7c4..5f294a3357305b 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1512,9 +1512,9 @@ mono_assembly_open_from_bundle (MonoAssemblyLoadContext *alc, const char *filena image = open_from_satellite_bundle (alc, filename, status, culture); } else { - const char *name = g_path_get_basename (filename); + char *name = g_path_get_basename (filename); image = open_from_bundle_internal (alc, name, status); - g_free ((void *)name); + g_free (name); } if (image) { mono_image_addref (image); From 50fd4ba79b268d38e597ceb26f9f10dadc800dd2 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 11 May 2023 18:24:52 -0400 Subject: [PATCH 055/110] Remove unnecessary includes --- src/mono/mono/metadata/bundled-resources-internals.h | 7 +------ src/mono/mono/metadata/bundled-resources.c | 7 ++----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 5f7e235d5fb4b1..dfae38faf19c95 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -5,13 +5,8 @@ #ifndef __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ #define __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ -#include #include - -#include "loader-internals.h" - -#include -#include +#include typedef enum { MONO_BUNDLED_DATA, diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index b490591c3aad38..3f23d0e4b0690f 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -5,11 +5,8 @@ #include #include -#include "bundled-resources-internals.h" -#include "assembly-internals.h" - -#include -#include +#include +#include static GHashTable *bundled_resources = NULL; static bool bundle_contains_assemblies, bundle_contains_satellite_assemblies = false; From 236539a55b3a5da3c9f1e68232105a8f68921d1d Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 11 May 2023 18:26:39 -0400 Subject: [PATCH 056/110] Fix trailing commas in structs and extra lines in generated files --- .../metadata/bundled-resources-internals.h | 2 +- src/tasks/EmitBundleTask/EmitBundleBase.cs | 31 +++++++++---------- .../Templates/mono-bundled-assembly.template | 4 +-- .../Templates/mono-bundled-data.template | 2 +- ...ce-preallocation-and-registration.template | 7 ++--- .../mono-bundled-satellite-assembly.template | 2 +- 6 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index dfae38faf19c95..a8dea132bc9a51 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -12,7 +12,7 @@ typedef enum { MONO_BUNDLED_DATA, MONO_BUNDLED_ASSEMBLY, MONO_BUNDLED_SATELLITE_ASSEMBLY, - MONO_BUNDLED_RESOURCE_COUNT, + MONO_BUNDLED_RESOURCE_COUNT } MonoBundledResourceType; typedef struct _MonoBundledResource { diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index fed2f2bd400c87..a211e1a5583ff4 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -216,16 +216,16 @@ private static byte[] InitLookupTable() private static void GenerateBundledResourcePreallocationAndRegistration(string resourceSymbols, string bundleRegistrationFunctionName, ICollection<(string registeredFilename, string resourceName, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) { - StringBuilder preallocatedSource = new(); + List preallocatedSource = new (); string assemblyTemplate = Utils.GetEmbeddedResource("mono-bundled-assembly.template"); string satelliteAssemblyTemplate = Utils.GetEmbeddedResource("mono-bundled-satellite-assembly.template"); string symbolDataTemplate = Utils.GetEmbeddedResource("mono-bundled-data.template"); var preallocatedResources = new StringBuilder(); - var preallocatedAssemblies = new StringBuilder("MonoBundledResource *bundledAssemblyResources[] = { "); - var preallocatedSatelliteAssemblies = new StringBuilder("MonoBundledResource *bundledSatelliteAssemblyResources[] = { "); - var preallocatedData = new StringBuilder("MonoBundledResource *bundledDataResources[] = { "); + List preallocatedAssemblies = new (); + List preallocatedSatelliteAssemblies = new (); + List preallocatedData = new (); int assembliesCount = 0; int satelliteAssembliesCount = 0; int dataCount = 0; @@ -240,7 +240,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r preloadedStruct = satelliteAssemblyTemplate; preloadedStruct.Replace("%Culture%", tuple.culture); resourceId = $"{tuple.culture}/{tuple.registeredFilename}"; - preallocatedSatelliteAssemblies.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); + preallocatedSatelliteAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); satelliteAssembliesCount += 1; } else if (tuple.registeredFilename.EndsWith(".dll")) @@ -250,22 +250,22 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r string preloadedSymbolData = ""; if (!string.IsNullOrEmpty(tuple.resourceSymbolName)) { - preloadedSymbolData = Utils.GetEmbeddedResource("mono-bundled-symbol.template") + preloadedSymbolData = $",\n{Utils.GetEmbeddedResource("mono-bundled-symbol.template") .Replace("%ResourceSymbolName%", tuple.resourceSymbolName) - .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName].ToString()); + .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName].ToString())}"; } preloadedStruct = preloadedStruct.Replace("%MonoBundledSymbolData%", preloadedSymbolData); - preallocatedAssemblies.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); + preallocatedAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); assembliesCount += 1; } else { preloadedStruct = symbolDataTemplate; - preallocatedData.Append($"(MonoBundledResource *)&{tuple.resourceName}, "); + preallocatedData.Add($" (MonoBundledResource *)&{tuple.resourceName}"); dataCount += 1; } - preallocatedSource.AppendLine(preloadedStruct.Replace("%ResourceName%", tuple.resourceName) + preallocatedSource.Add(preloadedStruct.Replace("%ResourceName%", tuple.resourceName) .Replace("%ResourceID%", resourceId) .Replace("%RegisteredFilename%", tuple.registeredFilename) .Replace("%Len%", $"{tuple.resourceName}_len_val")); @@ -273,24 +273,21 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r var addPreallocatedResources = new StringBuilder(); if (assembliesCount != 0) { - preallocatedAssemblies.AppendLine("};"); - preallocatedResources.AppendLine(preallocatedAssemblies.ToString()); + preallocatedResources.AppendLine($"MonoBundledResource *bundledAssemblyResources[] = {{\n{string.Join(",\n", preallocatedAssemblies)}\n}};"); addPreallocatedResources.AppendLine($" mono_bundled_resources_add (bundledAssemblyResources, {assembliesCount});"); } if (satelliteAssembliesCount != 0) { - preallocatedSatelliteAssemblies.AppendLine("};"); - preallocatedResources.AppendLine(preallocatedSatelliteAssemblies.ToString()); + preallocatedResources.AppendLine($"MonoBundledResource *bundledSatelliteAssemblyResources[] = {{\n{string.Join(",\n", preallocatedSatelliteAssemblies)}\n}};"); addPreallocatedResources.AppendLine($" mono_bundled_resources_add (bundledSatelliteAssemblyResources, {satelliteAssembliesCount});"); } if (dataCount != 0) { - preallocatedData.AppendLine("};"); - preallocatedResources.AppendLine(preallocatedData.ToString()); + preallocatedResources.AppendLine($"MonoBundledResource *bundledDataResources[] = {{\n{string.Join(",\n", preallocatedData)}\n}};"); addPreallocatedResources.AppendLine($" mono_bundled_resources_add (bundledDataResources, {dataCount});"); } outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-resource-preallocation-and-registration.template") .Replace("%ResourceSymbols%", resourceSymbols) - .Replace("%PreallocatedStructs%", preallocatedSource.ToString()) + .Replace("%PreallocatedStructs%", string.Join("\n", preallocatedSource)) .Replace("%PreallocatedResources%", preallocatedResources.ToString()) .Replace("%BundleRegistrationFunctionName%", bundleRegistrationFunctionName) .Replace("%AddPreallocatedResources%", addPreallocatedResources.ToString())); diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template index 88b17a21d4f688..231c8eb70a5a80 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template @@ -3,5 +3,5 @@ const MonoBundledAssemblyResource %ResourceName% = { .id = "%ResourceID%" }, .assembly = { .name = "%RegisteredFilename%", .data = %ResourceName%_data, - .size = %Len% }, -%MonoBundledSymbolData%}; \ No newline at end of file + .size = %Len% }%MonoBundledSymbolData% +}; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template index 2b2538733f9f8f..aa925a010a4fdc 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template @@ -3,5 +3,5 @@ const MonoBundledDataResource %ResourceName% = { .id = "%ResourceID%" }, .data = { .name = "%RegisteredFilename%", .data = %ResourceName%_data, - .size = %Len% }, + .size = %Len% } }; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template index 6c52956d30e37a..fd7ac2e183eaf9 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template @@ -4,12 +4,11 @@ #include %ResourceSymbols% - typedef enum { MONO_BUNDLED_DATA, MONO_BUNDLED_ASSEMBLY, MONO_BUNDLED_SATELLITE_ASSEMBLY, - MONO_BUNDLED_RESOURCE_COUNT, + MONO_BUNDLED_RESOURCE_COUNT } MonoBundledResourceType; typedef struct _MonoBundledResource { @@ -64,8 +63,6 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t %PreallocatedStructs% %PreallocatedResources% - void %BundleRegistrationFunctionName% (void) { -%AddPreallocatedResources% -} +%AddPreallocatedResources%} diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template index a239efc7c2a300..c2c42f443f1f1b 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template @@ -4,5 +4,5 @@ const MonoBundledSatelliteAssemblyResource %ResourceName% = { .satellite_assembly = { .name = "%RegisteredFilename%", .culture = "%Culture%", .data = %ResourceName%_data, - .size = %Len% }, + .size = %Len% } }; \ No newline at end of file From df96a6d8dfd7389075b04b7886d92b31f3646897 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 11 May 2023 18:27:18 -0400 Subject: [PATCH 057/110] Move specific bundled resource getters into source file --- .../metadata/bundled-resources-internals.h | 40 ++++--------------- src/mono/mono/metadata/bundled-resources.c | 33 +++++++++++++++ 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index a8dea132bc9a51..604492af8c5349 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -73,38 +73,14 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t MonoBundledResource * mono_bundled_resources_get (const char *id); -static inline MonoBundledAssemblyResource * -mono_bundled_resources_get_assembly_resource (const char *id) -{ - MonoBundledAssemblyResource *assembly = - (MonoBundledAssemblyResource*)mono_bundled_resources_get (id); - if (!assembly) - return NULL; - assert (assembly->resource.type == MONO_BUNDLED_ASSEMBLY); - return assembly; -} - -static inline MonoBundledSatelliteAssemblyResource * -mono_bundled_resources_get_satellite_assembly_resource (const char *id) -{ - MonoBundledSatelliteAssemblyResource *satellite_assembly = - (MonoBundledSatelliteAssemblyResource*)mono_bundled_resources_get (id); - if (!satellite_assembly) - return NULL; - assert (satellite_assembly->resource.type == MONO_BUNDLED_SATELLITE_ASSEMBLY); - return satellite_assembly; -} - -static inline MonoBundledDataResource * -mono_bundled_resources_get_data_resource (const char *id) -{ - MonoBundledDataResource *data = - (MonoBundledDataResource*)mono_bundled_resources_get (id); - if (!data) - return NULL; - assert (data->resource.type == MONO_BUNDLED_DATA); - return data; -} +MonoBundledAssemblyResource * +mono_bundled_resources_get_assembly_resource (const char *id); + +MonoBundledSatelliteAssemblyResource * +mono_bundled_resources_get_satellite_assembly_resource (const char *id); + +MonoBundledDataResource * +mono_bundled_resources_get_data_resource (const char *id); bool mono_bundled_resources_contains_assemblies (void); diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 3f23d0e4b0690f..f1b1301eaeabb1 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -145,6 +145,39 @@ mono_bundled_resources_get (const char *id) return g_hash_table_lookup (bundled_resources, id); } +MonoBundledAssemblyResource * +mono_bundled_resources_get_assembly_resource (const char *id) +{ + MonoBundledAssemblyResource *assembly = + (MonoBundledAssemblyResource*)mono_bundled_resources_get (id); + if (!assembly) + return NULL; + g_assert (assembly->resource.type == MONO_BUNDLED_ASSEMBLY); + return assembly; +} + +MonoBundledSatelliteAssemblyResource * +mono_bundled_resources_get_satellite_assembly_resource (const char *id) +{ + MonoBundledSatelliteAssemblyResource *satellite_assembly = + (MonoBundledSatelliteAssemblyResource*)mono_bundled_resources_get (id); + if (!satellite_assembly) + return NULL; + g_assert (satellite_assembly->resource.type == MONO_BUNDLED_SATELLITE_ASSEMBLY); + return satellite_assembly; +} + +MonoBundledDataResource * +mono_bundled_resources_get_data_resource (const char *id) +{ + MonoBundledDataResource *data = + (MonoBundledDataResource*)mono_bundled_resources_get (id); + if (!data) + return NULL; + g_assert (data->resource.type == MONO_BUNDLED_DATA); + return data; +} + //--------------------------------------------------------------------------------------- // // mono_bundled_resources_contains_assemblies returns whether or not assemblies From befa5d69b01b937f274108c5590525718fd3b4ad Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 11 May 2023 18:30:03 -0400 Subject: [PATCH 058/110] Various cleanups Instantiating boolean variables More direct insertion into hash table Removing unnecessary assertions Proper type casting Preferring symbol_data over symfile Adding void to prototype functions arg Fix indentation and spacing Prefer id over name --- src/mono/mono/metadata/bundled-resources.c | 11 ++++------- src/mono/mono/metadata/mono-debug.c | 8 +++----- src/mono/wasi/runtime/driver.c | 18 +++++++++--------- src/mono/wasm/runtime/driver.c | 4 ++-- src/native/libs/System.Native/pal_datetime.c | 2 +- ...rce-preallocation-and-registration.template | 2 +- src/tasks/LibraryBuilder/Templates/autoinit.c | 2 +- 7 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index f1b1301eaeabb1..50e3b0b363c886 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -89,27 +89,24 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t if (!bundled_resources) bundled_resources = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, mono_bundled_resources_value_destroy_func); - bool assemblyAdded, satelliteAssemblyAdded; + bool assemblyAdded, satelliteAssemblyAdded = false; for (uint32_t i = 0; i < len; ++i) { MonoBundledResource *resource_to_bundle = (MonoBundledResource *)resources_to_bundle[i]; switch (resource_to_bundle->type) { case MONO_BUNDLED_ASSEMBLY: { - MonoBundledAssemblyResource *assembly = (MonoBundledAssemblyResource *)resource_to_bundle; - g_hash_table_insert (bundled_resources, (gpointer) assembly->resource.id, assembly); + g_hash_table_insert (bundled_resources, (gpointer) resource_to_bundle->id, resource_to_bundle); assemblyAdded = true; break; } case MONO_BUNDLED_SATELLITE_ASSEMBLY: { - MonoBundledSatelliteAssemblyResource *satellite_assembly = (MonoBundledSatelliteAssemblyResource *)resource_to_bundle; - g_hash_table_insert (bundled_resources, (gpointer) satellite_assembly->resource.id, satellite_assembly); + g_hash_table_insert (bundled_resources, (gpointer) resource_to_bundle->id, resource_to_bundle); satelliteAssemblyAdded = true; break; } case MONO_BUNDLED_DATA: default: { - MonoBundledDataResource *data = (MonoBundledDataResource *)resource_to_bundle; - g_hash_table_insert (bundled_resources, (gpointer) data->resource.id, data); + g_hash_table_insert (bundled_resources, (gpointer) resource_to_bundle->id, resource_to_bundle); break; } } diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index dd271be3d9a597..73d5789dedc321 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1080,12 +1080,10 @@ mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte * assembly_resource->resource.id = assembly_name; assembly_resource->resource.free_bundled_resource_func = &mono_bundled_resources_free_bundled_resource_func; } else { - g_assert (assembly_resource->resource.type == MONO_BUNDLED_ASSEMBLY); - g_assert (!strcmp(assembly_resource->resource.id, assembly_name)); // Ensure the MonoBundledSymbolData has not been initialized g_assert (!assembly_resource->symbol_data.data && assembly_resource->symbol_data.size == 0); } - assembly_resource->symbol_data.data = (uint8_t *)raw_contents; + assembly_resource->symbol_data.data = (const uint8_t *)raw_contents; assembly_resource->symbol_data.size = (uint32_t)size; mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); } @@ -1107,8 +1105,8 @@ open_symfile_from_bundle (MonoImage *image) assembly = mono_bundled_resources_get_assembly_resource (module_name_dll_suffix); } g_free (module_name_dll_suffix); - if (assembly && assembly->symfile) - return mono_debug_open_image (image, assembly->symfile.data, assembly->symfile.size); + if (assembly && assembly->symbol_data) + return mono_debug_open_image (image, assembly->symbol_data.data, assembly->symbol_data.size); #endif return NULL; diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index b29d39995bedc2..415bf9c1632ce9 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -63,11 +63,11 @@ int32_t monoeg_g_hasenv(const char *variable); void mono_free (void*); int32_t mini_parse_debug_option (const char *option); char *mono_method_get_full_name (MonoMethod *method); -extern void mono_register_timezones_bundle(); +extern void mono_register_timezones_bundle (void); #ifdef WASM_SINGLE_FILE -extern void mono_register_assemblies_bundle(); +extern void mono_register_assemblies_bundle (void); #ifndef INVARIANT_GLOBALIZATION -extern void mono_register_icu_bundle(); +extern void mono_register_icu_bundle (void); #endif /* INVARIANT_GLOBALIZATION */ #endif /* WASM_SINGLE_FILE */ @@ -344,14 +344,14 @@ mono_wasm_register_bundled_satellite_assemblies (void) void load_icu_data (void) { #ifdef WASM_SINGLE_FILE - mono_register_icu_bundle(); + mono_register_icu_bundle (); - MonoBundledDataResource *icu_data = mono_bundled_resources_get_data_resource ("icudt.dat"); - if (!icu_data) { + MonoBundledDataResource *icu_data = mono_bundled_resources_get_data_resource ("icudt.dat"); + if (!icu_data) { printf("Could not load icudt.dat from the bundle"); assert(buffer); } - const unsigned char *buffer = icu_data->data.data; + const unsigned char *buffer = icu_data->data.data; #else /* WASM_SINGLE_FILE */ FILE *fileptr; unsigned char *buffer; @@ -443,9 +443,9 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mini_parse_debug_option ("top-runtime-invoke-unhandled"); - mono_register_timezones_bundle(); + mono_register_timezones_bundle (); #ifdef WASM_SINGLE_FILE - mono_register_assemblies_bundle(); + mono_register_assemblies_bundle (); #endif mono_dl_fallback_register (wasm_dl_load, wasm_dl_symbol, NULL, NULL); mono_wasm_install_get_native_to_interp_tramp (get_native_to_interp); diff --git a/src/mono/wasm/runtime/driver.c b/src/mono/wasm/runtime/driver.c index dc0d86d8e8650e..edb23669e40499 100644 --- a/src/mono/wasm/runtime/driver.c +++ b/src/mono/wasm/runtime/driver.c @@ -53,7 +53,7 @@ int monoeg_g_setenv(const char *variable, const char *value, int overwrite); int32_t mini_parse_debug_option (const char *option); char *mono_method_get_full_name (MonoMethod *method); -extern void mono_register_timezones_bundle(); +extern void mono_register_timezones_bundle (void); extern void mono_wasm_set_entrypoint_breakpoint (const char* assembly_name, int method_token); static void mono_wasm_init_finalizer_thread (void); @@ -512,7 +512,7 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mini_parse_debug_option ("top-runtime-invoke-unhandled"); - mono_register_timezones_bundle(); + mono_register_timezones_bundle (); mono_dl_fallback_register (wasm_dl_load, wasm_dl_symbol, NULL, NULL); mono_wasm_install_get_native_to_interp_tramp (get_native_to_interp); diff --git a/src/native/libs/System.Native/pal_datetime.c b/src/native/libs/System.Native/pal_datetime.c index 81355a706e4cc7..c1fb3e46f4a94b 100644 --- a/src/native/libs/System.Native/pal_datetime.c +++ b/src/native/libs/System.Native/pal_datetime.c @@ -21,7 +21,7 @@ static const int64_t TICKS_PER_MICROSECOND = 10; /* 1000 / 100 */ #endif #if defined(TARGET_WASI) || defined(TARGET_BROWSER) -extern MonoBundledResource * mono_get_bundled_resource_data (const char *name); +extern MonoBundledResource * mono_get_bundled_resource_data (const char *id); #endif // diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template index fd7ac2e183eaf9..047630554c652b 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template @@ -57,7 +57,7 @@ typedef struct _MonoBundledSatelliteAssemblyResource { MonoBundledSatelliteAssemblyData satellite_assembly; } MonoBundledSatelliteAssemblyResource; -void +extern void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); %PreallocatedStructs% diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index 3e45524114ff05..66574a20acc6c7 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -19,7 +19,7 @@ #include "library-builder.h" #if defined(BUNDLED_RESOURCES) -void mono_register_resources_bundle (void); +extern void mono_register_resources_bundle (void); extern uint8_t %RUNTIME_CONFIG_DATA%[]; extern const uint32_t %RUNTIME_CONFIG_DATA_LEN%; #endif From c3bb50a4cad413503e3cd4a7f2a58b1c2cf57b4e Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 11 May 2023 18:34:17 -0400 Subject: [PATCH 059/110] Cleanup logic surrounding EmitBundle task invocations --- src/mono/wasi/build/WasiApp.Native.targets | 17 ++++++++--------- src/mono/wasi/wasi.proj | 7 +++---- src/mono/wasm/wasm.proj | 9 ++++----- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 0df7aaf6d725e1..424dbb9792192f 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -349,8 +349,8 @@ - <_WasmAssembliesBundleObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_assemblies.o - <_WasmIcuBundleObjectFile>$(_WasmIntermediateOutputPath)wasi_bundled_icu.o + <_WasmAssembliesBundleObjectFile>wasi_bundled_assemblies.o + <_WasmIcuBundleObjectFile>wasi_bundled_icu.o @@ -371,20 +371,19 @@ FilesToBundle="@(WasmBundleAssembliesWithHashes)" ClangExecutable="$(WasiClang)" BundleName="mono_register_assemblies_bundle" - BundleFile="wasi_bundled_assemblies.o" + BundleFile="$(_WasmAssembliesBundleObjectFile)" OutputDirectory="$(_WasmIntermediateOutputPath)" /> - <_WasiObjectFilesForBundle Include="$(_WasmAssembliesBundleObjectFile)" /> + <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmAssembliesBundleObjectFile)" /> <_WasiObjectFilesForBundle Include="%(WasmBundleAssembliesWithHashes.DestinationFile)" /> - - + @@ -393,17 +392,17 @@ FilesToBundle="@(WasmBundleIcuWithHashes)" ClangExecutable="$(WasiClang)" BundleName="mono_register_icu_bundle" - BundleFile="wasi_bundled_icu.o" + BundleFile="$(_WasmIcuBundleObjectFile)" OutputDirectory="$(_WasmIntermediateOutputPath)" /> - <_WasiObjectFilesForBundle Include="$(_WasmIcuBundleObjectFile)" /> + <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmIcuBundleObjectFile)" /> <_WasiObjectFilesForBundle Include="%(WasmBundleIcuWithHashes.DestinationFile)" /> - + diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index 76db6961766a68..949a03d834a8ae 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -69,7 +69,7 @@ <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) - <_WasmTimezonesBundleObjectFile>$(WasiObjDir)\wasm-bundled-timezones.o + <_WasmTimezonesBundleObjectFile>wasm-bundled-timezones.o <_WasmTimezonesBundleArchive>$(WasiObjDir)\wasm-bundled-timezones.a <_WasmTimezonesArchiveRsp>$(WasiObjDir)\wasm-bundled-timezones-archive.rsp @@ -90,12 +90,12 @@ FilesToBundle="@(_WasmBundleTimezonesWithHashes)" ClangExecutable="$(WasiClang)" BundleName="mono_register_timezones_bundle" - BundleFile="wasm-bundled-timezones.o" + BundleFile="$(_WasmTimezonesBundleObjectFile)" OutputDirectory="$(WasiObjDir)" /> - + <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> - <_WasmBundleTimezonesToDelete Remove="$(WasiObjDir)mono-bundled-source.h" /> <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundlePreallocatedObjectFile)" /> <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index db2a348782184e..3ff292a69bdebe 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -84,7 +84,7 @@ <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) - <_WasmTimezonesBundleSourceFile>$(WasmObjDir)\wasm-bundled-timezones.c + <_WasmTimezonesBundleSourceFile>wasm-bundled-timezones.c <_WasmTimezonesBundleObjectFile>$(WasmObjDir)\wasm-bundled-timezones.o <_WasmTimezonesBundleArchive>$(WasmObjDir)\wasm-bundled-timezones.a <_WasmTimezonesSourcesRsp>$(WasmObjDir)\wasm-bundled-timezones-sources.rsp @@ -107,13 +107,13 @@ <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), %(_WasmBundleTimezonesWithHashes.DestinationFile)).Replace('\','/'))" /> - <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), $(_WasmTimezonesBundleSourceFile)).Replace('\','/'))" /> + <_WasmBundleTimezonesSources Include="$(_WasmTimezonesBundleSourceFile)" /> <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.c" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmIntermediateOutputPath)mono-bundled-source.h" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleSourceFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(WasmObjDir)\$(_WasmTimezonesBundleSourceFile)" /> <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.ObjectFile)" /> From a06074820d7c2faa08cb6b666edace1acd20caec Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 11 May 2023 18:35:03 -0400 Subject: [PATCH 060/110] Rename output metadata to DataLenSymbol and DataLenSymbolValue --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 5 ++--- src/tasks/LibraryBuilder/LibraryBuilder.cs | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index a211e1a5583ff4..57c9a4b30f8f60 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -155,15 +155,14 @@ public override bool Execute() resourceSymbols.AppendLine($"#define {symbolName}_len_val {symbolDataLen[symbolName]}"); contentSourceFile.SetMetadata("DataSymbol", $"{symbolName}_data"); - contentSourceFile.SetMetadata("LenSymbol", $"{symbolName}_len"); - contentSourceFile.SetMetadata("LenSymbolValue", symbolDataLen[symbolName].ToString()); + contentSourceFile.SetMetadata("DataLenSymbol", $"{symbolName}_len"); + contentSourceFile.SetMetadata("DataLenSymbolValue", symbolDataLen[symbolName].ToString()); bundledResources.Add(contentSourceFile); }); } BundledResources = bundledResources.ToArray(); if (!string.IsNullOrEmpty(BundleFile)) { - // Generate source file to preallocate resources and register bundled resources Emit(Path.Combine(OutputDirectory, BundleFile), (inputStream) => { diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index 1b9e24ec810469..0a2c0064ea3c56 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -297,9 +297,9 @@ private void WriteAutoInitializationFromTemplate() string dataSymbol = bundledRuntimeConfigInfo.GetMetadata("DataSymbol"); if (string.IsNullOrEmpty(dataSymbol)) throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataSymbol' metadata."); - string lenSymbol = bundledRuntimeConfigInfo.GetMetadata("LenSymbol"); + string lenSymbol = bundledRuntimeConfigInfo.GetMetadata("DataLenSymbol"); if (string.IsNullOrEmpty(lenSymbol)) - throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'LenSymbol' metadata."); + throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataLenSymbol' metadata."); autoInitialization = autoInitialization .Replace("%RUNTIME_CONFIG_DATA%", dataSymbol) From aff9c60fc5d137ba4403925feb958afbf42f0774 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 11 May 2023 21:29:07 -0400 Subject: [PATCH 061/110] Simplify bundled resource hashtable insertion --- src/mono/mono/metadata/bundled-resources.c | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 50e3b0b363c886..3c1d1989f5f8a4 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -93,23 +93,13 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t for (uint32_t i = 0; i < len; ++i) { MonoBundledResource *resource_to_bundle = (MonoBundledResource *)resources_to_bundle[i]; - switch (resource_to_bundle->type) { - case MONO_BUNDLED_ASSEMBLY: { - g_hash_table_insert (bundled_resources, (gpointer) resource_to_bundle->id, resource_to_bundle); + if (resource_to_bundle->type == MONO_BUNDLED_ASSEMBLY) assemblyAdded = true; - break; - } - case MONO_BUNDLED_SATELLITE_ASSEMBLY: { - g_hash_table_insert (bundled_resources, (gpointer) resource_to_bundle->id, resource_to_bundle); + + if (resource_to_bundle->type == MONO_BUNDLED_SATELLITE_ASSEMBLY) satelliteAssemblyAdded = true; - break; - } - case MONO_BUNDLED_DATA: - default: { - g_hash_table_insert (bundled_resources, (gpointer) resource_to_bundle->id, resource_to_bundle); - break; - } - } + + g_hash_table_insert (bundled_resources, (gpointer) resource_to_bundle->id, resource_to_bundle); } if (assemblyAdded) From c1e9ec1a045f9ae3225c1e7bbc4e7c339bfb71e1 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 12 May 2023 20:31:51 -0400 Subject: [PATCH 062/110] Cleanup variable instantiation --- src/mono/mono/metadata/bundled-resources.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 3c1d1989f5f8a4..6577b7f117b5ac 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -9,7 +9,8 @@ #include static GHashTable *bundled_resources = NULL; -static bool bundle_contains_assemblies, bundle_contains_satellite_assemblies = false; +static bool bundle_contains_assemblies = false; +static bool bundle_contains_satellite_assemblies = false; //--------------------------------------------------------------------------------------- // @@ -89,7 +90,8 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t if (!bundled_resources) bundled_resources = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, mono_bundled_resources_value_destroy_func); - bool assemblyAdded, satelliteAssemblyAdded = false; + bool assemblyAdded = false; + bool satelliteAssemblyAdded = false; for (uint32_t i = 0; i < len; ++i) { MonoBundledResource *resource_to_bundle = (MonoBundledResource *)resources_to_bundle[i]; From aeaf8791d69233dd4d1f37928b5dcc9eaa679758 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 12 May 2023 20:32:15 -0400 Subject: [PATCH 063/110] Fixup mono-debug --- src/mono/mono/metadata/mono-debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 73d5789dedc321..8c530b6479b08b 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1105,7 +1105,7 @@ open_symfile_from_bundle (MonoImage *image) assembly = mono_bundled_resources_get_assembly_resource (module_name_dll_suffix); } g_free (module_name_dll_suffix); - if (assembly && assembly->symbol_data) + if (assembly && assembly->symbol_data.data) return mono_debug_open_image (image, assembly->symbol_data.data, assembly->symbol_data.size); #endif From a6b92455716540664ab165514e004571fc2ef708 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 12 May 2023 20:32:46 -0400 Subject: [PATCH 064/110] Cleanup header --- src/mono/wasi/runtime/driver.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index 415bf9c1632ce9..d66f593c681460 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -35,7 +36,6 @@ #endif #include "gc-common.h" #include "driver.h" -#include "bundled-resources-internals.h" #if !defined(ENABLE_AOT) || defined(EE_MODE_LLVMONLY_INTERP) From 6c7d33df1e744aa18c860e803c72ea313d6f7e13 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 12 May 2023 20:33:05 -0400 Subject: [PATCH 065/110] fixup wasi --- src/mono/wasi/wasi.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index 949a03d834a8ae..a0d4906a37f464 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -110,7 +110,7 @@ <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundlePreallocatedObjectFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(WasiObjDir)\$(_WasmTimezonesBundleObjectFile)" /> <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> From dfba3f477bdb0695cc3b0cd68ceca4f1128be2b6 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 12 May 2023 20:34:36 -0400 Subject: [PATCH 066/110] fixup template format --- ...no-bundled-resource-preallocation-and-registration.template | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template index 047630554c652b..1352e44c047354 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template @@ -64,5 +64,6 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t %PreallocatedResources% void -%BundleRegistrationFunctionName% (void) { +%BundleRegistrationFunctionName% (void) +{ %AddPreallocatedResources%} From e8aa970e9cf6322f57d661518946988a697e94ff Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 12 May 2023 20:36:15 -0400 Subject: [PATCH 067/110] Add flexibility to auto initialize runtime without bundled runtimeconfig --- src/tasks/LibraryBuilder/LibraryBuilder.cs | 39 ++++++++++++------- src/tasks/LibraryBuilder/Templates/autoinit.c | 7 ++-- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index 0a2c0064ea3c56..f7eb26d4a36cdc 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -91,9 +91,12 @@ public bool IsSharedLibrary public bool BundlesResources { get; set; } /// - /// An Item containing the bundled runtimeconfig.bin metadata populated by EmitBundle task + /// An Item containing the bundled runtimeconfig.bin metadata detailing + /// DataSymbol - Symbol corresponding to the runtimeconfig.bin byte array data + /// DataLenSymbol - Symbol corresponding to the runtimeconfig.bin byte array size + /// DataLenSymbolValue - Literal size of the runtimeconfig.bin byte array data /// - public ITaskItem[] BundledRuntimeConfig { get; set; } = default!; + public ITaskItem? BundledRuntimeConfig { get; set; } public bool StripDebugSymbols { get; set; } @@ -155,10 +158,6 @@ public override bool Execute() if (UsesRuntimeInitCallback && !UsesCustomRuntimeInitCallback) { - if (BundlesResources && BundledRuntimeConfig.Length == 0) - { - throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' is required when bundling and using the default runtime callback."); - } WriteAutoInitializationFromTemplate(); extraSources.AppendLine(" autoinit.c"); } @@ -293,17 +292,29 @@ private void WriteAutoInitializationFromTemplate() if (BundlesResources) { - var bundledRuntimeConfigInfo = BundledRuntimeConfig.First(); - string dataSymbol = bundledRuntimeConfigInfo.GetMetadata("DataSymbol"); - if (string.IsNullOrEmpty(dataSymbol)) - throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataSymbol' metadata."); - string lenSymbol = bundledRuntimeConfigInfo.GetMetadata("DataLenSymbol"); - if (string.IsNullOrEmpty(lenSymbol)) - throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataLenSymbol' metadata."); + string dataSymbol = "NULL"; + string dataLenSymbol = "0"; + StringBuilder externRuntimeConfigSymbols = new (); + if (BundledRuntimeConfig?.ItemSpec != null) + { + dataSymbol = BundledRuntimeConfig.GetMetadata("DataSymbol"); + if (string.IsNullOrEmpty(dataSymbol)) + { + throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataSymbol' metadata."); + } + dataLenSymbol = BundledRuntimeConfig.GetMetadata("DataLenSymbol"); + if (string.IsNullOrEmpty(dataLenSymbol)) + { + throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataLenSymbol' metadata."); + } + externRuntimeConfigSymbols.AppendLine($"extern uint8_t {dataSymbol}[];"); + externRuntimeConfigSymbols.AppendLine($"extern const uint32_t {dataLenSymbol}[];"); + } autoInitialization = autoInitialization + .Replace("%EXTERN_RUNTIMECONFIG_SYMBOLS%", externRuntimeConfigSymbols.ToString()) .Replace("%RUNTIME_CONFIG_DATA%", dataSymbol) - .Replace("%RUNTIME_CONFIG_DATA_LEN%", lenSymbol); + .Replace("%RUNTIME_CONFIG_DATA_LEN%", dataLenSymbol); } File.WriteAllText(Path.Combine(OutputDirectory, "autoinit.c"), autoInitialization); diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index 66574a20acc6c7..7e71b43165f874 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -20,9 +20,7 @@ #if defined(BUNDLED_RESOURCES) extern void mono_register_resources_bundle (void); -extern uint8_t %RUNTIME_CONFIG_DATA%[]; -extern const uint32_t %RUNTIME_CONFIG_DATA_LEN%; -#endif +%EXTERN_RUNTIMECONFIG_SYMBOLS%#endif static void cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) @@ -45,6 +43,9 @@ initialize_runtimeconfig (const char *bundle_path) arg->kind = 1; arg->runtimeconfig.data.data = %RUNTIME_CONFIG_DATA%; arg->runtimeconfig.data.data_len = %RUNTIME_CONFIG_DATA_LEN%; + + if (!arg->runtimeconfig.data.data) + return; #else size_t str_len = sizeof (char) * (strlen (bundle_path) + strlen (file_name) + 2); // +1 "/", +1 null-terminating char char *file_path = (char *)malloc (str_len); From d191d59ccd61bd9943f44fc5e6db7063d5b047b3 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 12 May 2023 21:31:19 -0400 Subject: [PATCH 068/110] Various fixups --- src/mono/mono/metadata/assembly.c | 14 ++++++-------- src/mono/mono/metadata/bundled-resources.c | 1 - src/mono/msbuild/common/LibraryBuilder.targets | 4 ---- src/native/libs/System.Native/pal_datetime.c | 7 ++++++- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 5f294a3357305b..15b65a0e65f569 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -18,7 +18,7 @@ #include #include #include "assembly-internals.h" -#include "bundled-resources-internals.h" +#include #include #include "image-internals.h" #include "object-internals.h" @@ -1508,14 +1508,11 @@ mono_assembly_open_from_bundle (MonoAssemblyLoadContext *alc, const char *filena */ MonoImage *image = NULL; MONO_ENTER_GC_UNSAFE; - if (culture && culture [0] != 0) { + if (culture && culture [0] != 0) image = open_from_satellite_bundle (alc, filename, status, culture); - } - else { - char *name = g_path_get_basename (filename); - image = open_from_bundle_internal (alc, name, status); - g_free (name); - } + else + image = open_from_bundle_internal (alc, filename, status); + if (image) { mono_image_addref (image); mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_ASSEMBLY, "Assembly Loader loaded assembly from bundle: '%s'.", filename); @@ -3188,6 +3185,7 @@ mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly * char *id = g_strconcat (satellite_assembly->culture, "/", satellite_assembly->name, (const char*)NULL); MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); g_assert (!satellite_assembly_resource); + satellite_assembly_resource = g_new0 (MonoBundledSatelliteAssemblyResource, 1); satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; satellite_assembly_resource->resource.id = id; satellite_assembly_resource->resource.free_bundled_resource_func = mono_bundled_resources_free_bundled_resource_func; diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 6577b7f117b5ac..bac22abc2b71a4 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -6,7 +6,6 @@ #include #include -#include static GHashTable *bundled_resources = NULL; static bool bundle_contains_assemblies = false; diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index 63a272328bf3df..4c6b1d45d68b91 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -24,8 +24,6 @@ <_BundledRuntimeConfig Include="@(_BundledResources)" Condition="$([System.String]::Copy('%(Identity)').EndsWith('runtimeconfig.bin'))" /> - - - - data.size; return timezoneData->data.data; #else From afaf91afae9c4f53fafc2c55d00f806f46941cff Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Mon, 15 May 2023 16:23:59 -0400 Subject: [PATCH 069/110] Use g_free for memory allocated through glib apis --- src/mono/mono/metadata/bundled-resources.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index bac22abc2b71a4..cb553883d5c72f 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -45,9 +45,9 @@ mono_bundled_resources_free_bundled_resource_func (void *resource) MonoBundledResource *bundled_resource = (MonoBundledResource *)resource; if (bundled_resource->type == MONO_BUNDLED_SATELLITE_ASSEMBLY) - free ((void *)bundled_resource->id); + g_free ((void *)bundled_resource->id); - free (resource); + g_free (resource); } //--------------------------------------------------------------------------------------- From 3eff6b95eb88f1b601305c604aa502e5bb333d81 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Mon, 15 May 2023 16:25:25 -0400 Subject: [PATCH 070/110] Move condition end tag into LibraryBuilder to avoid empty line and cluttered line --- src/tasks/LibraryBuilder/LibraryBuilder.cs | 2 ++ src/tasks/LibraryBuilder/Templates/autoinit.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index f7eb26d4a36cdc..ceab0222976fff 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -311,6 +311,8 @@ private void WriteAutoInitializationFromTemplate() externRuntimeConfigSymbols.AppendLine($"extern const uint32_t {dataLenSymbol}[];"); } + externRuntimeConfigSymbols.AppendLine("#endif"); + autoInitialization = autoInitialization .Replace("%EXTERN_RUNTIMECONFIG_SYMBOLS%", externRuntimeConfigSymbols.ToString()) .Replace("%RUNTIME_CONFIG_DATA%", dataSymbol) diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index 7e71b43165f874..cb3e5ac1774a32 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -20,7 +20,7 @@ #if defined(BUNDLED_RESOURCES) extern void mono_register_resources_bundle (void); -%EXTERN_RUNTIMECONFIG_SYMBOLS%#endif +%EXTERN_RUNTIMECONFIG_SYMBOLS% static void cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) From 7d3bebdd234875d1f0698eb41adbc175dc569bfa Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Mon, 15 May 2023 16:36:45 -0400 Subject: [PATCH 071/110] Add bundled resource getters to access data values without MonoBundledResource structs --- .../metadata/bundled-resources-internals.h | 9 +++ src/mono/mono/metadata/bundled-resources.c | 57 +++++++++++++++++++ src/mono/wasi/runtime/driver.c | 6 +- src/native/libs/System.Native/pal_datetime.c | 14 ++--- 4 files changed, 73 insertions(+), 13 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 604492af8c5349..8692de670e28ea 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -82,6 +82,15 @@ mono_bundled_resources_get_satellite_assembly_resource (const char *id); MonoBundledDataResource * mono_bundled_resources_get_data_resource (const char *id); +bool +mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out, const uint8_t **symbol_data_out, uint32_t *symbol_size_out); + +bool +mono_bundled_resources_get_satellite_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); + +bool +mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); + bool mono_bundled_resources_contains_assemblies (void); diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index cb553883d5c72f..5077a834c7cb9a 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -166,6 +166,63 @@ mono_bundled_resources_get_data_resource (const char *id) return data; } +bool +mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out, const uint8_t **symbol_data_out, uint32_t *symbol_size_out) +{ + *data_out = NULL; + *size_out = 0; + *symbol_data_out = NULL; + *size_out = 0; + MonoBundledAssemblyResource *bundled_assembly_resource = mono_bundled_resources_get_assembly_resource (id); + if (!bundled_assembly_resource || + !bundled_assembly_resource->assembly.data || + bundled_assembly_resource->assembly.size == 0) + return false; + + *data_out = bundled_assembly_resource->assembly.data; + *size_out = bundled_assembly_resource->assembly.size; + *symbol_data_out = bundled_assembly_resource->symbol_data.data; + *symbol_size_out = bundled_assembly_resource->symbol_data.size; + + return true; +} + +bool +mono_bundled_resources_get_satellite_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out) +{ + *data_out = NULL; + *size_out = 0; + + MonoBundledSatelliteAssemblyResource *bundled_satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); + if (!bundled_satellite_assembly_resource || + !bundled_satellite_assembly_resource->satellite_assembly.data || + bundled_satellite_assembly_resource->satellite_assembly.size == 0) + return false; + + *data_out = bundled_satellite_assembly_resource->satellite_assembly.data; + *size_out = bundled_satellite_assembly_resource->satellite_assembly.size; + + return true; +} + +bool +mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out) +{ + *data_out = NULL; + *size_out = 0; + + MonoBundledDataResource *bundled_data_resource = mono_bundled_resources_get_data_resource (id); + if (!bundled_data_resource || + !bundled_data_resource->data.data || + bundled_data_resource->data.size == 0) + return false; + + *data_out = bundled_data_resource->data.data; + *size_out = bundled_data_resource->data.size; + + return true; +} + //--------------------------------------------------------------------------------------- // // mono_bundled_resources_contains_assemblies returns whether or not assemblies diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index d66f593c681460..9290bdab021645 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -346,12 +346,12 @@ void load_icu_data (void) #ifdef WASM_SINGLE_FILE mono_register_icu_bundle (); - MonoBundledDataResource *icu_data = mono_bundled_resources_get_data_resource ("icudt.dat"); - if (!icu_data) { + unsigned char *buffer = NULL; + int data_len = 0; + if (!mono_bundled_resources_get_data_resource_values ("icudt.dat", &buffer, &data_len)) { printf("Could not load icudt.dat from the bundle"); assert(buffer); } - const unsigned char *buffer = icu_data->data.data; #else /* WASM_SINGLE_FILE */ FILE *fileptr; unsigned char *buffer; diff --git a/src/native/libs/System.Native/pal_datetime.c b/src/native/libs/System.Native/pal_datetime.c index 1691846b9fbd9a..04f752377c5004 100644 --- a/src/native/libs/System.Native/pal_datetime.c +++ b/src/native/libs/System.Native/pal_datetime.c @@ -21,7 +21,7 @@ static const int64_t TICKS_PER_MICROSECOND = 10; /* 1000 / 100 */ #endif #if defined(TARGET_WASI) || defined(TARGET_BROWSER) -extern MonoBundledResource * mono_get_bundled_resource_data (const char *id); +extern bool mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); #endif // @@ -67,15 +67,9 @@ const char* SystemNative_GetTimeZoneData(const char* name, int* length) assert(name != NULL); assert(length != NULL); #if defined(TARGET_WASI) || defined(TARGET_BROWSER) - MonoBundledDataResource *timezoneData = (MonoBundledDataResource *)mono_get_bundled_resource_data (name); - if (!timezoneData) - { - *length = 0; - return NULL; - } - - length = timezoneData->data.size; - return timezoneData->data.data; + unsigned char *data = NULL; + mono_bundled_resources_get_data_resource_values (name, &data, length); + return data; #else assert_msg(false, "Not supported on this platform", 0); (void)name; // unused From 187855f1533aa10f8bb6c6e3f05880f05a9bbfcd Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Mon, 15 May 2023 23:59:30 -0400 Subject: [PATCH 072/110] Emit one object file for single resource data source file mode --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 153 +++++------ .../WasmAppBuilder/EmitWasmBundleBase.cs | 240 ------------------ 2 files changed, 81 insertions(+), 312 deletions(-) delete mode 100644 src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 57c9a4b30f8f60..fbe80adb20cd0a 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -73,6 +73,78 @@ public override bool Execute() var verbose = remainingDestinationFilesToBundle.Length > 1; var verboseCount = 0; + List bundledResources = new(remainingDestinationFilesToBundle.Length); + // Generate source file(s) containing each resource's byte data and size + if (remainingDestinationFilesToBundle.Length > 0) + { + if (string.IsNullOrEmpty(CombinedResourceSource)) + { + int allowedParallelism = Math.Max(Math.Min(remainingDestinationFilesToBundle.Length, Environment.ProcessorCount), 1); + if (BuildEngine is IBuildEngine9 be9) + allowedParallelism = be9.RequestCores(allowedParallelism); + Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => + { + var group = remainingDestinationFilesToBundle[i]; + + // Since the object filenames include a content hash, we can pick an arbitrary ITaskItem from each group, + // since we know each group's ITaskItems all contain the same binary data + var contentSourceFile = group.First(); + + var outputFile = group.Key; + var inputFile = contentSourceFile.ItemSpec; + + if (verbose) + { + var registeredName = contentSourceFile.GetMetadata("RegisteredName"); + if(string.IsNullOrEmpty(registeredName)) + { + registeredName = Path.GetFileName(inputFile); + } + var count = Interlocked.Increment(ref verboseCount); + Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); + } + + Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, outputFile); + var symbolName = ToSafeSymbolName(group.Key); + if (!Emit(outputFile, (codeStream) => { + using var inputStream = File.OpenRead(inputFile); + using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); + BundleFileToCSource(symbolName, true, inputStream, outputUtf8Writer); + })) + { + state.Stop(); + } + + contentSourceFile.SetMetadata("DataSymbol", $"{symbolName}_data"); + contentSourceFile.SetMetadata("DataLenSymbol", $"{symbolName}_len"); + contentSourceFile.SetMetadata("DataLenSymbolValue", symbolDataLen[symbolName].ToString()); + bundledResources.Add(contentSourceFile); + }); + } + else + { + Emit(Path.Combine(OutputDirectory, CombinedResourceSource), (codeStream) => { + using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); + bool shouldAddHeader = true; + foreach (var destinationFileGroup in remainingDestinationFilesToBundle) + { + ITaskItem destinationFile = destinationFileGroup.First(); + var symbolName = ToSafeSymbolName(destinationFileGroup.Key); + using var inputStream = File.OpenRead(destinationFile.ItemSpec); + Console.WriteLine($"destinationFile: {destinationFile.ItemSpec}"); + BundleFileToCSource(symbolName, shouldAddHeader, inputStream, outputUtf8Writer); + shouldAddHeader = false; + + destinationFile.SetMetadata("DataSymbol", $"{symbolName}_data"); + destinationFile.SetMetadata("DataLenSymbol", $"{symbolName}_len"); + destinationFile.SetMetadata("DataLenSymbolValue", symbolDataLen[symbolName].ToString()); + bundledResources.Add(destinationFile); + } + }); + } + } + + var resourceSymbols = new StringBuilder(); var filesToBundleByRegisteredName = FilesToBundle.GroupBy(file => { var registeredName = file.GetMetadata("RegisteredName"); if(string.IsNullOrEmpty(registeredName)) @@ -89,6 +161,9 @@ public override bool Execute() var resourceName = ToSafeSymbolName(outputFile); string culture = registeredFile.GetMetadata("Culture"); string? resourceSymbolName = null; + resourceSymbols.AppendLine($"extern uint8_t {resourceName}_data[];"); + resourceSymbols.AppendLine($"extern const uint32_t {resourceName}_len;"); + resourceSymbols.AppendLine($"#define {resourceName}_len_val {symbolDataLen[resourceName]}"); if (File.Exists(registeredFile.GetMetadata("SymbolFile"))) resourceSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("SymbolFile")); return (registeredFilename, resourceName, culture, resourceSymbolName); @@ -96,77 +171,13 @@ public override bool Execute() Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); - HashSet resourceSourcesGenerated = new(); - List bundledResources = new(remainingDestinationFilesToBundle.Length); - var resourceSymbols = new StringBuilder(); - // Generate source file(s) containing each resource's byte data and size - if (remainingDestinationFilesToBundle.Length > 0) - { - int allowedParallelism = Math.Max(Math.Min(remainingDestinationFilesToBundle.Length, Environment.ProcessorCount), 1); - if (BuildEngine is IBuildEngine9 be9) - allowedParallelism = be9.RequestCores(allowedParallelism); - if (!string.IsNullOrEmpty(CombinedResourceSource)) - allowedParallelism = 1; - - Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => - { - var group = remainingDestinationFilesToBundle[i]; - - // Since the object filenames include a content hash, we can pick an arbitrary ITaskItem from each group, - // since we know each group's ITaskItems all contain the same binary data - var contentSourceFile = group.First(); - - var outputFile = group.Key; - if (!string.IsNullOrEmpty(CombinedResourceSource)) - outputFile = Path.Combine(OutputDirectory, CombinedResourceSource); - var inputFile = contentSourceFile.ItemSpec; - bool shouldAddHeader = false; - lock (resourceSourcesGenerated) - { - shouldAddHeader = !resourceSourcesGenerated.Contains(outputFile); - } - - if (verbose) - { - var registeredName = contentSourceFile.GetMetadata("RegisteredName"); - if(string.IsNullOrEmpty(registeredName)) - { - registeredName = Path.GetFileName(inputFile); - } - var count = Interlocked.Increment(ref verboseCount); - Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); - } - - Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, outputFile); - var symbolName = ToSafeSymbolName(group.Key); - if (!Emit(outputFile, (codeStream) => { - using var inputStream = File.OpenRead(inputFile); - BundleFileToCSource(symbolName, shouldAddHeader, inputStream, codeStream); - })) - { - state.Stop(); - } - lock (resourceSourcesGenerated) - { - resourceSourcesGenerated.Add(outputFile); - } - resourceSymbols.AppendLine($"extern uint8_t {symbolName}_data[];"); - resourceSymbols.AppendLine($"extern const uint32_t {symbolName}_len;"); - resourceSymbols.AppendLine($"#define {symbolName}_len_val {symbolDataLen[symbolName]}"); - - contentSourceFile.SetMetadata("DataSymbol", $"{symbolName}_data"); - contentSourceFile.SetMetadata("DataLenSymbol", $"{symbolName}_len"); - contentSourceFile.SetMetadata("DataLenSymbolValue", symbolDataLen[symbolName].ToString()); - bundledResources.Add(contentSourceFile); - }); - } BundledResources = bundledResources.ToArray(); if (!string.IsNullOrEmpty(BundleFile)) { // Generate source file to preallocate resources and register bundled resources - Emit(Path.Combine(OutputDirectory, BundleFile), (inputStream) => + Emit(Path.Combine(OutputDirectory, BundleFile), (outputStream) => { - using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); + using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); GenerateBundledResourcePreallocationAndRegistration(resourceSymbols.ToString(), BundleRegistrationFunctionName, files, outputUtf8Writer); }); } @@ -292,7 +303,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r .Replace("%AddPreallocatedResources%", addPreallocatedResources.ToString())); } - private static void BundleFileToCSource(string symbolName, bool shouldAddHeader, FileStream inputStream, Stream outputStream) + private static void BundleFileToCSource(string symbolName, bool shouldAddHeader, FileStream inputStream, StreamWriter outputUtf8Writer) { // Emits a C source file in the same format as "xxd --include". Example: // @@ -306,8 +317,6 @@ private static void BundleFileToCSource(string symbolName, bool shouldAddHeader, var generatedArrayLength = 0; var bytesEmitted = 0; - using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); - if (shouldAddHeader) outputUtf8Writer.WriteLine("#include "); @@ -319,11 +328,11 @@ private static void BundleFileToCSource(string symbolName, bool shouldAddHeader, { if (bytesEmitted++ % 12 == 0) { - outputStream.Write(NewLineAndIndentation, 0, NewLineAndIndentation.Length); + outputUtf8Writer.BaseStream.Write(NewLineAndIndentation, 0, NewLineAndIndentation.Length); } var byteValue = buf[i]; - outputStream.Write(HexToUtf8Lookup, byteValue * 6, 6); + outputUtf8Writer.BaseStream.Write(HexToUtf8Lookup, byteValue * 6, 6); } generatedArrayLength += bytesRead; @@ -332,7 +341,7 @@ private static void BundleFileToCSource(string symbolName, bool shouldAddHeader, outputUtf8Writer.WriteLine("0\n};"); outputUtf8Writer.WriteLine($"const uint32_t {symbolName}_len = {generatedArrayLength};"); outputUtf8Writer.Flush(); - outputStream.Flush(); + outputUtf8Writer.BaseStream.Flush(); symbolDataLen.Add(symbolName, generatedArrayLength); } diff --git a/src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs b/src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs deleted file mode 100644 index fb6e897d7ff3d4..00000000000000 --- a/src/tasks/WasmAppBuilder/EmitWasmBundleBase.cs +++ /dev/null @@ -1,240 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Build.Framework; - -namespace Microsoft.WebAssembly.Build.Tasks; - -public abstract class EmitWasmBundleBase : Microsoft.Build.Utilities.Task, ICancelableTask -{ - private CancellationTokenSource BuildTaskCancelled { get; } = new(); - - /// Must have DestinationFile metadata, which is the output filename - /// Could have RegisteredName, otherwise it would be the filename. - /// RegisteredName should be prefixed with namespace in form of unix like path. For example: "/usr/share/zoneinfo/" - [Required] - public ITaskItem[] FilesToBundle { get; set; } = default!; - - [Required] - public string BundleName { get; set; } = default!; - - [Required] - public string BundleFile { get; set; } = default!; - - [Required] - public string RegistrationCallbackFunctionName { get; set; } = default!; - - public override bool Execute() - { - // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore - // produces one group per file-content. We only want to emit one copy of each file-content, and one symbol for it. - var filesToBundleByDestinationFileName = FilesToBundle.GroupBy(f => f.GetMetadata("DestinationFile")).ToList(); - - // We're handling the incrementalism within this task, because it needs to be based on file content hashes - // and not on timetamps. The output filenames contain a content hash, so if any such file already exists on - // disk with that name, we know it must be up-to-date. - var remainingDestinationFilesToBundle = filesToBundleByDestinationFileName.Where(g => !File.Exists(g.Key)).ToArray(); - - // If you're only touching the leaf project, we don't really need to tell you that. - // But if there's more work to do it's valuable to show progress. - var verbose = remainingDestinationFilesToBundle.Length > 1; - var verboseCount = 0; - - var filesToBundleByRegisteredName = FilesToBundle.GroupBy(file => { - var registeredName = file.GetMetadata("RegisteredName"); - if(string.IsNullOrEmpty(registeredName)) - { - registeredName = Path.GetFileName(file.ItemSpec); - } - return registeredName; - }).ToList(); - - var files = filesToBundleByRegisteredName.Select(group => { - var registeredFile = group.First(); - var outputFile = registeredFile.GetMetadata("DestinationFile"); - var registeredName = group.Key; - var symbolName = ToSafeSymbolName(outputFile); - return (registeredName, symbolName); - }).ToList(); - - Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleName}"); - - if (remainingDestinationFilesToBundle.Length > 0) - { - int allowedParallelism = Math.Max(Math.Min(remainingDestinationFilesToBundle.Length, Environment.ProcessorCount), 1); - if (BuildEngine is IBuildEngine9 be9) - allowedParallelism = be9.RequestCores(allowedParallelism); - - Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => - { - var group = remainingDestinationFilesToBundle[i]; - - // Since the object filenames include a content hash, we can pick an arbitrary ITaskItem from each group, - // since we know each group's ITaskItems all contain the same binary data - var contentSourceFile = group.First(); - - var outputFile = group.Key; - var inputFile = contentSourceFile.ItemSpec; - if (verbose) - { - var registeredName = contentSourceFile.GetMetadata("RegisteredName"); - if(string.IsNullOrEmpty(registeredName)) - { - registeredName = Path.GetFileName(inputFile); - } - var count = Interlocked.Increment(ref verboseCount); - Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); - } - - Log.LogMessage(MessageImportance.Low, "Bundling {0} as {1}", inputFile, outputFile); - var symbolName = ToSafeSymbolName(outputFile); - if (!Emit(outputFile, (codeStream) => { - using var inputStream = File.OpenRead(inputFile); - BundleFileToCSource(symbolName, inputStream, codeStream); - })) - { - state.Stop(); - } - }); - } - - return Emit(BundleFile, (inputStream) => - { - using var outputUtf8Writer = new StreamWriter(inputStream, Utf8NoBom); - GenerateRegisteredBundledObjects($"mono_wasm_register_{BundleName}_bundle", RegistrationCallbackFunctionName, files, outputUtf8Writer); - }) && !Log.HasLoggedErrors; - } - - public void Cancel() - { - BuildTaskCancelled.Cancel(); - } - - #region Helpers - - private static readonly Encoding Utf8NoBom = new UTF8Encoding(false); - private static readonly byte[] HexToUtf8Lookup = InitLookupTable(); - private static readonly byte[] NewLineAndIndentation = new[] { (byte)0x0a, (byte)0x20, (byte)0x20 }; - - private static byte[] InitLookupTable() - { - // Every 6 bytes in this array represents the output for a different input byte value. - // For example, the input byte 0x1a (26 decimal) corresponds to bytes 156-161 (26*6=156), - // whose values will be ['0', 'x', '1', 'a', ',', ' '], which is the UTF-8 representation - // for "0x1a, ". This is just a faster alternative to calling .ToString("x2") on every - // byte of the input file and then pushing that string through UTF8Encoding. - var lookup = new byte[256 * 6]; - for (int i = 0; i < 256; i++) - { - string byteAsHex = i.ToString("x2"); - char highOrderChar = BitConverter.IsLittleEndian ? byteAsHex[0] : byteAsHex[1]; - char lowOrderChar = BitConverter.IsLittleEndian ? byteAsHex[1] : byteAsHex[0]; - lookup[i * 6 + 0] = (byte)'0'; - lookup[i * 6 + 1] = (byte)'x'; - lookup[i * 6 + 2] = (byte)highOrderChar; - lookup[i * 6 + 3] = (byte)lowOrderChar; - lookup[i * 6 + 4] = (byte)','; - lookup[i * 6 + 5] = (byte)' '; - } - - return lookup; - } - - public abstract bool Emit(string destinationFile, Action inputProvider); - - public static void GenerateRegisteredBundledObjects(string newFunctionName, string callbackFunctionName, ICollection<(string registeredName, string symbol)> files, StreamWriter outputUtf8Writer) - { - outputUtf8Writer.WriteLine($"int {callbackFunctionName}(const char* name, const unsigned char* data, unsigned int size);"); - outputUtf8Writer.WriteLine(); - - foreach (var tuple in files) - { - outputUtf8Writer.WriteLine($"extern const unsigned char {tuple.symbol}[];"); - outputUtf8Writer.WriteLine($"extern const int {tuple.symbol}_len;"); - } - - outputUtf8Writer.WriteLine(); - outputUtf8Writer.WriteLine($"void {newFunctionName}() {{"); - - foreach (var tuple in files) - { - outputUtf8Writer.WriteLine($" {callbackFunctionName} (\"{tuple.registeredName}\", {tuple.symbol}, {tuple.symbol}_len);"); - } - - outputUtf8Writer.WriteLine("}"); - } - - private static void BundleFileToCSource(string symbolName, FileStream inputStream, Stream outputStream) - { - // Emits a C source file in the same format as "xxd --include". Example: - // - // unsigned char Some_File_dll[] = { - // 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x0a - // }; - // unsigned int Some_File_dll_len = 6; - - var buf = new byte[4096]; - int bytesRead; - var generatedArrayLength = 0; - var bytesEmitted = 0; - - using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); - - outputUtf8Writer.Write($"unsigned char {symbolName}[] = {{"); - outputUtf8Writer.Flush(); - while ((bytesRead = inputStream.Read(buf, 0, buf.Length)) > 0) - { - for (var i = 0; i < bytesRead; i++) - { - if (bytesEmitted++ % 12 == 0) - { - outputStream.Write(NewLineAndIndentation, 0, NewLineAndIndentation.Length); - } - - var byteValue = buf[i]; - outputStream.Write(HexToUtf8Lookup, byteValue * 6, 6); - } - - generatedArrayLength += bytesRead; - } - - outputUtf8Writer.WriteLine("0\n};"); - outputUtf8Writer.WriteLine($"unsigned int {symbolName}_len = {generatedArrayLength};"); - outputUtf8Writer.Flush(); - outputStream.Flush(); - } - - private static string ToSafeSymbolName(string destinationFileName) - { - // Since destinationFileName includes a content hash, we can safely strip off the directory name - // as the filename is always unique enough. This avoid disclosing information about the build - // file structure in the resulting symbols. - var filename = Path.GetFileName(destinationFileName); - - // Equivalent to the logic from "xxd --include" - var sb = new StringBuilder(); - foreach (var c in filename) - { - sb.Append(IsAlphanumeric(c) ? c : '_'); - } - - return sb.ToString(); - } - - // Equivalent to "isalnum" - private static bool IsAlphanumeric(char c) => c - is (>= 'a' and <= 'z') - or (>= 'A' and <= 'Z') - or (>= '0' and <= '9'); - - #endregion - -} From 7cf6edd189c28e22e0591eb04b64a6cff5620051 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 16 May 2023 00:58:29 -0400 Subject: [PATCH 073/110] Rename resource size symbol --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 12 ++++++------ src/tasks/LibraryBuilder/LibraryBuilder.cs | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index fbe80adb20cd0a..1ddc5a0cd83cbe 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -116,7 +116,7 @@ public override bool Execute() } contentSourceFile.SetMetadata("DataSymbol", $"{symbolName}_data"); - contentSourceFile.SetMetadata("DataLenSymbol", $"{symbolName}_len"); + contentSourceFile.SetMetadata("DataLenSymbol", $"{symbolName}_data_len"); contentSourceFile.SetMetadata("DataLenSymbolValue", symbolDataLen[symbolName].ToString()); bundledResources.Add(contentSourceFile); }); @@ -136,7 +136,7 @@ public override bool Execute() shouldAddHeader = false; destinationFile.SetMetadata("DataSymbol", $"{symbolName}_data"); - destinationFile.SetMetadata("DataLenSymbol", $"{symbolName}_len"); + destinationFile.SetMetadata("DataLenSymbol", $"{symbolName}_data_len"); destinationFile.SetMetadata("DataLenSymbolValue", symbolDataLen[symbolName].ToString()); bundledResources.Add(destinationFile); } @@ -162,8 +162,8 @@ public override bool Execute() string culture = registeredFile.GetMetadata("Culture"); string? resourceSymbolName = null; resourceSymbols.AppendLine($"extern uint8_t {resourceName}_data[];"); - resourceSymbols.AppendLine($"extern const uint32_t {resourceName}_len;"); - resourceSymbols.AppendLine($"#define {resourceName}_len_val {symbolDataLen[resourceName]}"); + resourceSymbols.AppendLine($"extern const uint32_t {resourceName}_data_len;"); + resourceSymbols.AppendLine($"#define {resourceName}_data_len_val {symbolDataLen[resourceName]}"); if (File.Exists(registeredFile.GetMetadata("SymbolFile"))) resourceSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("SymbolFile")); return (registeredFilename, resourceName, culture, resourceSymbolName); @@ -278,7 +278,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r preallocatedSource.Add(preloadedStruct.Replace("%ResourceName%", tuple.resourceName) .Replace("%ResourceID%", resourceId) .Replace("%RegisteredFilename%", tuple.registeredFilename) - .Replace("%Len%", $"{tuple.resourceName}_len_val")); + .Replace("%Len%", $"{tuple.resourceName}_data_len_val")); } var addPreallocatedResources = new StringBuilder(); @@ -339,7 +339,7 @@ private static void BundleFileToCSource(string symbolName, bool shouldAddHeader, } outputUtf8Writer.WriteLine("0\n};"); - outputUtf8Writer.WriteLine($"const uint32_t {symbolName}_len = {generatedArrayLength};"); + outputUtf8Writer.WriteLine($"const uint32_t {symbolName}_data_len = {generatedArrayLength};"); outputUtf8Writer.Flush(); outputUtf8Writer.BaseStream.Flush(); diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index ceab0222976fff..008e4744df3aed 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -308,7 +308,7 @@ private void WriteAutoInitializationFromTemplate() throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataLenSymbol' metadata."); } externRuntimeConfigSymbols.AppendLine($"extern uint8_t {dataSymbol}[];"); - externRuntimeConfigSymbols.AppendLine($"extern const uint32_t {dataLenSymbol}[];"); + externRuntimeConfigSymbols.AppendLine($"extern const uint32_t {dataLenSymbol};"); } externRuntimeConfigSymbols.AppendLine("#endif"); From 8a78222e317cd026373575673d5318e738e8c0a3 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 16 May 2023 11:21:47 -0400 Subject: [PATCH 074/110] Use public friendly API in file linking towards runtime --- src/mono/wasi/runtime/driver.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index 9290bdab021645..2e7f3c7d2d9aa3 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -11,7 +11,6 @@ #include #include -#include #include #include #include @@ -67,6 +66,7 @@ extern void mono_register_timezones_bundle (void); #ifdef WASM_SINGLE_FILE extern void mono_register_assemblies_bundle (void); #ifndef INVARIANT_GLOBALIZATION +extern bool mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); extern void mono_register_icu_bundle (void); #endif /* INVARIANT_GLOBALIZATION */ #endif /* WASM_SINGLE_FILE */ From f5ec4288a9372cbb829fd277a02617eb00fcdee1 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 16 May 2023 11:22:50 -0400 Subject: [PATCH 075/110] Cleanup types populated through bundled resource getters --- src/mono/wasi/runtime/driver.c | 4 ++-- src/native/libs/System.Native/pal_datetime.c | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index 2e7f3c7d2d9aa3..c0cac4a47c3709 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -346,8 +346,8 @@ void load_icu_data (void) #ifdef WASM_SINGLE_FILE mono_register_icu_bundle (); - unsigned char *buffer = NULL; - int data_len = 0; + const uint8_t *buffer = NULL; + uint32_t data_len = 0; if (!mono_bundled_resources_get_data_resource_values ("icudt.dat", &buffer, &data_len)) { printf("Could not load icudt.dat from the bundle"); assert(buffer); diff --git a/src/native/libs/System.Native/pal_datetime.c b/src/native/libs/System.Native/pal_datetime.c index 04f752377c5004..94a05ebb63e694 100644 --- a/src/native/libs/System.Native/pal_datetime.c +++ b/src/native/libs/System.Native/pal_datetime.c @@ -4,6 +4,7 @@ #include "pal_config.h" #include "pal_datetime.h" #include "pal_utilities.h" +#include #include #include #include @@ -67,9 +68,18 @@ const char* SystemNative_GetTimeZoneData(const char* name, int* length) assert(name != NULL); assert(length != NULL); #if defined(TARGET_WASI) || defined(TARGET_BROWSER) - unsigned char *data = NULL; - mono_bundled_resources_get_data_resource_values (name, &data, length); - return data; + const uint8_t *data = NULL; + uint32_t data_len = 0; + + mono_bundled_resources_get_data_resource_values (name, &data, &data_len); + assert (data_len <= INT_MAX); + if (data_len > INT_MAX) { + data_len = 0; + data = NULL; + } + + *length = (int)data_len; + return (const char *)data; #else assert_msg(false, "Not supported on this platform", 0); (void)name; // unused From f3095698913b5708d99fe1239fbabe49b030de9c Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 16 May 2023 11:24:00 -0400 Subject: [PATCH 076/110] Move extern block completely to LibraryBuilder side --- src/tasks/LibraryBuilder/LibraryBuilder.cs | 2 +- src/tasks/LibraryBuilder/Templates/autoinit.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index 008e4744df3aed..531525ef8c777d 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -294,7 +294,7 @@ private void WriteAutoInitializationFromTemplate() { string dataSymbol = "NULL"; string dataLenSymbol = "0"; - StringBuilder externRuntimeConfigSymbols = new (); + StringBuilder externRuntimeConfigSymbols = new ("#if defined(BUNDLED_RESOURCES)\nextern void mono_register_resources_bundle (void);"); if (BundledRuntimeConfig?.ItemSpec != null) { dataSymbol = BundledRuntimeConfig.GetMetadata("DataSymbol"); diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index cb3e5ac1774a32..92d50ff6c88299 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -18,8 +18,6 @@ #include "library-builder.h" -#if defined(BUNDLED_RESOURCES) -extern void mono_register_resources_bundle (void); %EXTERN_RUNTIMECONFIG_SYMBOLS% static void From 5001622f1894ecc9fcb20e4929bfd399b47adb4a Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 16 May 2023 12:10:10 -0400 Subject: [PATCH 077/110] Update EmitBundle task invocations to use BundleRegistrationFunctionName --- src/mono/wasi/build/WasiApp.Native.targets | 4 ++-- src/mono/wasi/wasi.proj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 424dbb9792192f..65721a06cb3076 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -370,7 +370,7 @@ @@ -391,7 +391,7 @@ Condition="'$(InvariantGlobalization)' != 'true'" FilesToBundle="@(WasmBundleIcuWithHashes)" ClangExecutable="$(WasiClang)" - BundleName="mono_register_icu_bundle" + BundleRegistrationFunctionName="mono_register_icu_bundle" BundleFile="$(_WasmIcuBundleObjectFile)" OutputDirectory="$(_WasmIntermediateOutputPath)" /> diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index a0d4906a37f464..8a0eca0fe44548 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -89,7 +89,7 @@ From 0b3b30dabe3633b602cc486f18d1b07ee8060dbf Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 16 May 2023 23:29:09 -0400 Subject: [PATCH 078/110] Refactor EmitBundle to account for timezone duplicate resource contents --- src/tasks/EmitBundleTask/EmitBundleBase.cs | 231 +++++++++--------- .../Templates/mono-bundled-assembly.template | 2 +- .../Templates/mono-bundled-data.template | 2 +- .../mono-bundled-satellite-assembly.template | 2 +- 4 files changed, 125 insertions(+), 112 deletions(-) diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/EmitBundleTask/EmitBundleBase.cs index 1ddc5a0cd83cbe..b2172460f28bec 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/EmitBundleTask/EmitBundleBase.cs @@ -61,124 +61,111 @@ public override bool Execute() { // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore // produces one group per file-content. We only want to emit one copy of each file-content, and one symbol for it. - var filesToBundleByDestinationFileName = FilesToBundle.GroupBy(f => f.GetMetadata("DestinationFile")).ToList(); + var remainingDestinationFilesToBundle = FilesToBundle.GroupBy(f => f.GetMetadata("DestinationFile")).ToArray(); - // We're handling the incrementalism within this task, because it needs to be based on file content hashes - // and not on timetamps. The output filenames contain a content hash, so if any such file already exists on - // disk with that name, we know it must be up-to-date. - var remainingDestinationFilesToBundle = filesToBundleByDestinationFileName.Where(g => !File.Exists(g.Key)).ToArray(); - - // If you're only touching the leaf project, we don't really need to tell you that. - // But if there's more work to do it's valuable to show progress. - var verbose = remainingDestinationFilesToBundle.Length > 1; var verboseCount = 0; - List bundledResources = new(remainingDestinationFilesToBundle.Length); // Generate source file(s) containing each resource's byte data and size - if (remainingDestinationFilesToBundle.Length > 0) + if (string.IsNullOrEmpty(CombinedResourceSource)) { - if (string.IsNullOrEmpty(CombinedResourceSource)) + int allowedParallelism = Math.Max(Math.Min(remainingDestinationFilesToBundle.Length, Environment.ProcessorCount), 1); + if (BuildEngine is IBuildEngine9 be9) + allowedParallelism = be9.RequestCores(allowedParallelism); + + Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => { - int allowedParallelism = Math.Max(Math.Min(remainingDestinationFilesToBundle.Length, Environment.ProcessorCount), 1); - if (BuildEngine is IBuildEngine9 be9) - allowedParallelism = be9.RequestCores(allowedParallelism); - Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => + var group = remainingDestinationFilesToBundle[i]; + + // Since the object filenames include a content hash, we can pick an arbitrary ITaskItem from each group, + // since we know each group's ITaskItems all contain the same binary data + var contentSourceFile = group.First(); + + var outputFile = group.Key; + var inputFile = contentSourceFile.ItemSpec; + + var registeredName = contentSourceFile.GetMetadata("RegisteredName"); + if(string.IsNullOrEmpty(registeredName)) { - var group = remainingDestinationFilesToBundle[i]; - - // Since the object filenames include a content hash, we can pick an arbitrary ITaskItem from each group, - // since we know each group's ITaskItems all contain the same binary data - var contentSourceFile = group.First(); - - var outputFile = group.Key; - var inputFile = contentSourceFile.ItemSpec; - - if (verbose) - { - var registeredName = contentSourceFile.GetMetadata("RegisteredName"); - if(string.IsNullOrEmpty(registeredName)) - { - registeredName = Path.GetFileName(inputFile); - } - var count = Interlocked.Increment(ref verboseCount); - Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); - } - - Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, outputFile); - var symbolName = ToSafeSymbolName(group.Key); - if (!Emit(outputFile, (codeStream) => { - using var inputStream = File.OpenRead(inputFile); - using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); - BundleFileToCSource(symbolName, true, inputStream, outputUtf8Writer); - })) - { - state.Stop(); - } - - contentSourceFile.SetMetadata("DataSymbol", $"{symbolName}_data"); - contentSourceFile.SetMetadata("DataLenSymbol", $"{symbolName}_data_len"); - contentSourceFile.SetMetadata("DataLenSymbolValue", symbolDataLen[symbolName].ToString()); - bundledResources.Add(contentSourceFile); - }); - } - else - { - Emit(Path.Combine(OutputDirectory, CombinedResourceSource), (codeStream) => { + registeredName = Path.GetFileName(inputFile); + } + var count = Interlocked.Increment(ref verboseCount); + Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); + + Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, outputFile); + var symbolName = ToSafeSymbolName(outputFile); + if (!Emit(outputFile, (codeStream) => { + using var inputStream = File.OpenRead(inputFile); using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); - bool shouldAddHeader = true; - foreach (var destinationFileGroup in remainingDestinationFilesToBundle) - { - ITaskItem destinationFile = destinationFileGroup.First(); - var symbolName = ToSafeSymbolName(destinationFileGroup.Key); - using var inputStream = File.OpenRead(destinationFile.ItemSpec); - Console.WriteLine($"destinationFile: {destinationFile.ItemSpec}"); - BundleFileToCSource(symbolName, shouldAddHeader, inputStream, outputUtf8Writer); - shouldAddHeader = false; - - destinationFile.SetMetadata("DataSymbol", $"{symbolName}_data"); - destinationFile.SetMetadata("DataLenSymbol", $"{symbolName}_data_len"); - destinationFile.SetMetadata("DataLenSymbolValue", symbolDataLen[symbolName].ToString()); - bundledResources.Add(destinationFile); - } - }); - } + BundleFileToCSource(symbolName, true, inputStream, outputUtf8Writer); + })) + { + state.Stop(); + } + }); + } + else + { + Emit(Path.Combine(OutputDirectory, CombinedResourceSource), (codeStream) => { + using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); + bool shouldAddHeader = true; + foreach (var destinationFileGroup in remainingDestinationFilesToBundle) + { + ITaskItem destinationFile = destinationFileGroup.First(); + var symbolName = ToSafeSymbolName(destinationFileGroup.Key); + using var inputStream = File.OpenRead(destinationFile.ItemSpec); + BundleFileToCSource(symbolName, shouldAddHeader, inputStream, outputUtf8Writer); + shouldAddHeader = false; + } + }); } - var resourceSymbols = new StringBuilder(); - var filesToBundleByRegisteredName = FilesToBundle.GroupBy(file => { - var registeredName = file.GetMetadata("RegisteredName"); - if(string.IsNullOrEmpty(registeredName)) - { - registeredName = Path.GetFileName(file.ItemSpec); - } - return registeredName; - }).ToList(); - - var files = filesToBundleByRegisteredName.Select(group => { - var registeredFile = group.First(); - var outputFile = registeredFile.GetMetadata("DestinationFile"); - var registeredFilename = group.Key; - var resourceName = ToSafeSymbolName(outputFile); - string culture = registeredFile.GetMetadata("Culture"); - string? resourceSymbolName = null; - resourceSymbols.AppendLine($"extern uint8_t {resourceName}_data[];"); - resourceSymbols.AppendLine($"extern const uint32_t {resourceName}_data_len;"); - resourceSymbols.AppendLine($"#define {resourceName}_data_len_val {symbolDataLen[resourceName]}"); - if (File.Exists(registeredFile.GetMetadata("SymbolFile"))) - resourceSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("SymbolFile")); - return (registeredFilename, resourceName, culture, resourceSymbolName); - }).ToList(); - - Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); - + List bundledResources = new(FilesToBundle.Length); + foreach (ITaskItem file in FilesToBundle) + { + ITaskItem bundledResource = file; + string resourceDataSymbol = ToSafeSymbolName(bundledResource.GetMetadata("DestinationFile")); + bundledResource.SetMetadata("DataSymbol", $"{resourceDataSymbol}_data"); + bundledResource.SetMetadata("DataLenSymbol", $"{resourceDataSymbol}_data_len"); + bundledResource.SetMetadata("DataLenSymbolValue", symbolDataLen[resourceDataSymbol].ToString()); + bundledResources.Add(bundledResource); + } BundledResources = bundledResources.ToArray(); - if (!string.IsNullOrEmpty(BundleFile)) { + if (!string.IsNullOrEmpty(BundleFile)) + { + var filesToBundleByRegisteredName = FilesToBundle.GroupBy(file => { + var registeredName = file.GetMetadata("RegisteredName"); + if(string.IsNullOrEmpty(registeredName)) + { + registeredName = Path.GetFileName(file.ItemSpec); + } + return registeredName; + }).ToList(); + + string resourceSymbols = GatherUniqueExportedResourceDataSymbols(remainingDestinationFilesToBundle); + + var files = filesToBundleByRegisteredName.Select(group => { + ITaskItem registeredFile = group.First(); + var registeredFilename = group.Key; + var resourceName = ToSafeSymbolName(registeredFilename, false); + // Different timezone resources may have the same contents, use registered name to differentiate preallocated resources + var resourceDataSymbol = ToSafeSymbolName(registeredFile.GetMetadata("DestinationFile")); + + string culture = registeredFile.GetMetadata("Culture"); + string? resourceSymbolName = null; + if (File.Exists(registeredFile.GetMetadata("SymbolFile"))) + resourceSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("SymbolFile")); + + return (registeredFilename, resourceName, resourceDataSymbol, culture, resourceSymbolName); + }).ToList(); + + Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); + // Generate source file to preallocate resources and register bundled resources Emit(Path.Combine(OutputDirectory, BundleFile), (outputStream) => { using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); - GenerateBundledResourcePreallocationAndRegistration(resourceSymbols.ToString(), BundleRegistrationFunctionName, files, outputUtf8Writer); + GenerateBundledResourcePreallocationAndRegistration(resourceSymbols, BundleRegistrationFunctionName, files, outputUtf8Writer); }); } @@ -224,7 +211,25 @@ private static byte[] InitLookupTable() private static Dictionary symbolDataLen = new(); - private static void GenerateBundledResourcePreallocationAndRegistration(string resourceSymbols, string bundleRegistrationFunctionName, ICollection<(string registeredFilename, string resourceName, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) + private static string GatherUniqueExportedResourceDataSymbols(IEnumerable> uniqueDestinationFiles) + { + StringBuilder resourceSymbols = new (); + HashSet resourcesAdded = new (); // Different Timezone resources may have the same contents + foreach (var uniqueDestinationFile in uniqueDestinationFiles) + { + string resourceDataSymbol = ToSafeSymbolName(uniqueDestinationFile.Key); + if (!resourcesAdded.Contains(resourceDataSymbol)) + { + resourceSymbols.AppendLine($"extern uint8_t {resourceDataSymbol}_data[];"); + resourceSymbols.AppendLine($"extern const uint32_t {resourceDataSymbol}_data_len;"); + resourceSymbols.AppendLine($"#define {resourceDataSymbol}_data_len_val {symbolDataLen[resourceDataSymbol]}"); + resourcesAdded.Add(resourceDataSymbol); + } + } + return resourceSymbols.ToString(); + } + + private static void GenerateBundledResourcePreallocationAndRegistration(string resourceSymbols, string bundleRegistrationFunctionName, ICollection<(string registeredFilename, string resourceName, string resourceDataSymbol, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) { List preallocatedSource = new (); @@ -276,9 +281,10 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r } preallocatedSource.Add(preloadedStruct.Replace("%ResourceName%", tuple.resourceName) + .Replace("%ResourceDataSymbol%", tuple.resourceDataSymbol) .Replace("%ResourceID%", resourceId) .Replace("%RegisteredFilename%", tuple.registeredFilename) - .Replace("%Len%", $"{tuple.resourceName}_data_len_val")); + .Replace("%Len%", $"{tuple.resourceDataSymbol}_data_len_val")); } var addPreallocatedResources = new StringBuilder(); @@ -303,7 +309,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r .Replace("%AddPreallocatedResources%", addPreallocatedResources.ToString())); } - private static void BundleFileToCSource(string symbolName, bool shouldAddHeader, FileStream inputStream, StreamWriter outputUtf8Writer) + private void BundleFileToCSource(string symbolName, bool shouldAddHeader, FileStream inputStream, StreamWriter outputUtf8Writer) { // Emits a C source file in the same format as "xxd --include". Example: // @@ -343,21 +349,28 @@ private static void BundleFileToCSource(string symbolName, bool shouldAddHeader, outputUtf8Writer.Flush(); outputUtf8Writer.BaseStream.Flush(); - symbolDataLen.Add(symbolName, generatedArrayLength); + lock (symbolDataLen) + { + if (!symbolDataLen.TryAdd(symbolName, generatedArrayLength) && symbolDataLen[symbolName] != generatedArrayLength) + Log.LogMessage(MessageImportance.High, $"There are duplicate resources with the same output symbol '{symbolName}' but have differing content sizes '{symbolDataLen[symbolName]}' != '{generatedArrayLength}'."); + } } - private static string ToSafeSymbolName(string destinationFileName) + private static string ToSafeSymbolName(string destinationFileName, bool filenameOnly = true) { // Since destinationFileName includes a content hash, we can safely strip off the directory name // as the filename is always unique enough. This avoid disclosing information about the build // file structure in the resulting symbols. - var filename = Path.GetFileName(destinationFileName); + var filename = destinationFileName; + if (filenameOnly) + filename = Path.GetFileName(destinationFileName); // Equivalent to the logic from "xxd --include" var sb = new StringBuilder(); foreach (var c in filename) { - sb.Append(IsAlphanumeric(c) ? c : '_'); + sb.Append(IsAlphanumeric(c) ? c : + (c == '+') ? "plus" : '_'); // To help differentiate timezones differing by a symbol (i.e. GMT+0 GMT-0) } return sb.ToString(); diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template index 231c8eb70a5a80..a2ff6570a50110 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template @@ -2,6 +2,6 @@ const MonoBundledAssemblyResource %ResourceName% = { .resource = { .type = MONO_BUNDLED_ASSEMBLY, .id = "%ResourceID%" }, .assembly = { .name = "%RegisteredFilename%", - .data = %ResourceName%_data, + .data = %ResourceDataSymbol%_data, .size = %Len% }%MonoBundledSymbolData% }; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template index aa925a010a4fdc..03a9b43823be5f 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template @@ -2,6 +2,6 @@ const MonoBundledDataResource %ResourceName% = { .resource = { .type = MONO_BUNDLED_DATA, .id = "%ResourceID%" }, .data = { .name = "%RegisteredFilename%", - .data = %ResourceName%_data, + .data = %ResourceDataSymbol%_data, .size = %Len% } }; \ No newline at end of file diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template index c2c42f443f1f1b..1fbee8bbf2bf18 100644 --- a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template +++ b/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template @@ -3,6 +3,6 @@ const MonoBundledSatelliteAssemblyResource %ResourceName% = { .id = "%ResourceID%" }, .satellite_assembly = { .name = "%RegisteredFilename%", .culture = "%Culture%", - .data = %ResourceName%_data, + .data = %ResourceDataSymbol%_data, .size = %Len% } }; \ No newline at end of file From 2af674d1b48fdea161bb37e9aceeb2a4054e96c5 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 19 May 2023 18:51:57 -0400 Subject: [PATCH 079/110] wip --- src/mono/mono/metadata/assembly.c | 2 +- src/mono/mono/metadata/mono-debug.c | 2 +- .../msbuild/apple/build/AppleBuild.LocalBuild.props | 3 +++ .../apple/build/AppleBuild.LocalBuild.targets | 1 + src/tasks/EmitBundleTask/EmitBundleBase.cs | 12 ++++++------ src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs | 2 +- src/tasks/LibraryBuilder/LibraryBuilder.cs | 10 +++++----- src/tasks/LibraryBuilder/Templates/autoinit.c | 2 +- 8 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index fb2f159cd9675b..91cb5f7b07e4fa 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -3161,6 +3161,7 @@ mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; assembly_resource->resource.id = assembly->name; assembly_resource->resource.free_bundled_resource_func = mono_bundled_resources_free_bundled_resource_func; + mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); } else { // Ensure the MonoBundledAssemblyData has not been initialized g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); @@ -3168,7 +3169,6 @@ mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) assembly_resource->assembly.name = assembly->name; assembly_resource->assembly.data = (const uint8_t *)assembly->data; assembly_resource->assembly.size = (uint32_t)assembly->size; - mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); } } diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index a18d504597ceae..1b62f9b5bf9be5 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1080,13 +1080,13 @@ mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte * assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; assembly_resource->resource.id = assembly_name; assembly_resource->resource.free_bundled_resource_func = &mono_bundled_resources_free_bundled_resource_func; + mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); } else { // Ensure the MonoBundledSymbolData has not been initialized g_assert (!assembly_resource->symbol_data.data && assembly_resource->symbol_data.size == 0); } assembly_resource->symbol_data.data = (const uint8_t *)raw_contents; assembly_resource->symbol_data.size = (uint32_t)size; - mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); } static MonoDebugHandle * diff --git a/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.props b/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.props index 0c66b9186ee63e..7bd72e454b1e90 100644 --- a/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.props +++ b/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.props @@ -37,6 +37,7 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'LibraryBuilder', 'Debug', '$(_NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(_NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoTargetsTasks', 'Debug', '$(_NetCoreAppToolCurrent)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'EmitBundle', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'mono', '$(TargetOS).$(TargetArchitecture).$(RuntimeConfig)')) $([MSBuild]::NormalizePath($(MonoArtifactsPath), 'cross', '$(TargetOS)-$(TargetArchitecture.ToLowerInvariant())')) @@ -54,6 +55,7 @@ $([MSBuild]::NormalizeDirectory($(BuildBaseDir), 'MonoTargetsTasks')) $([MSBuild]::NormalizeDirectory($(BuildBaseDir), 'AppleAppBuilder')) $([MSBuild]::NormalizeDirectory($(BuildBaseDir), 'LibraryBuilder')) + $([MSBuild]::NormalizeDirectory('$(BuildBaseDir)', 'EmitBundle')) $([MSBuild]::NormalizePath($(BuildBaseDir), 'cross')) <_MonoAotCrossCompilerPath>$([MSBuild]::NormalizePath($(MonoAotCrossDir), 'mono-aot-cross')) @@ -71,6 +73,7 @@ $([MSBuild]::NormalizePath('$(LibraryBuilderDir)', 'LibraryBuilder.dll')) $([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll')) $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll')) + $([MSBuild]::NormalizePath('$(EmitBundleDir)', 'EmitBundle.dll')) diff --git a/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.targets b/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.targets index 05bdce89bf8432..705d9a37a635aa 100644 --- a/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.targets +++ b/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.targets @@ -62,6 +62,7 @@ $([MSBuild]::NormalizePath('$(LibraryBuilderDir)', 'LibraryBuilder.dll')) $([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll')) $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'RuntimeConfigParser.dll')) + $([MSBuild]::NormalizePath('$(EmitBundleDir)', 'EmitBundle.dll')) inputProvider) { - using (var fileStream = new FileStream(destinationFile, FileMode.Append, FileAccess.Write)) + using (var fileStream = File.Create(destinationFile)) { inputProvider(fileStream); } diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index 531525ef8c777d..b66d4838319588 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -294,7 +294,7 @@ private void WriteAutoInitializationFromTemplate() { string dataSymbol = "NULL"; string dataLenSymbol = "0"; - StringBuilder externRuntimeConfigSymbols = new ("#if defined(BUNDLED_RESOURCES)\nextern void mono_register_resources_bundle (void);"); + StringBuilder externBundledResourcesSymbols = new ("#if defined(BUNDLED_RESOURCES)\nextern void mono_register_resources_bundle (void);"); if (BundledRuntimeConfig?.ItemSpec != null) { dataSymbol = BundledRuntimeConfig.GetMetadata("DataSymbol"); @@ -307,14 +307,14 @@ private void WriteAutoInitializationFromTemplate() { throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataLenSymbol' metadata."); } - externRuntimeConfigSymbols.AppendLine($"extern uint8_t {dataSymbol}[];"); - externRuntimeConfigSymbols.AppendLine($"extern const uint32_t {dataLenSymbol};"); + externBundledResourcesSymbols.AppendLine($"extern uint8_t {dataSymbol}[];"); + externBundledResourcesSymbols.AppendLine($"extern const uint32_t {dataLenSymbol};"); } - externRuntimeConfigSymbols.AppendLine("#endif"); + externBundledResourcesSymbols.AppendLine("#endif"); autoInitialization = autoInitialization - .Replace("%EXTERN_RUNTIMECONFIG_SYMBOLS%", externRuntimeConfigSymbols.ToString()) + .Replace("%EXTERN_BUNDLED_RESOURCES_SYMBOLS%", externBundledResourcesSymbols.ToString()) .Replace("%RUNTIME_CONFIG_DATA%", dataSymbol) .Replace("%RUNTIME_CONFIG_DATA_LEN%", dataLenSymbol); } diff --git a/src/tasks/LibraryBuilder/Templates/autoinit.c b/src/tasks/LibraryBuilder/Templates/autoinit.c index 92d50ff6c88299..e104345a434085 100644 --- a/src/tasks/LibraryBuilder/Templates/autoinit.c +++ b/src/tasks/LibraryBuilder/Templates/autoinit.c @@ -18,7 +18,7 @@ #include "library-builder.h" -%EXTERN_RUNTIMECONFIG_SYMBOLS% +%EXTERN_BUNDLED_RESOURCES_SYMBOLS% static void cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data) From 4541530abcf741211ed6f69970ce32004dd9ca3c Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 23 May 2023 15:21:22 -0400 Subject: [PATCH 080/110] wip --- src/mono/mono/metadata/assembly.c | 7 ++++-- src/mono/wasi/build/WasiApp.Native.targets | 2 +- src/mono/wasi/runtime/driver.c | 1 + src/mono/wasi/wasi.proj | 2 +- src/mono/wasm/wasm.proj | 2 +- src/tasks/EmitBundleTask/EmitBundle.csproj | 24 ------------------- .../EmitBundleTask/EmitBundleBase.cs | 14 ++++++++--- .../EmitBundleTask/EmitBundleObjectFiles.cs | 0 .../EmitBundleTask/EmitBundleSourceFiles.cs | 0 .../MonoTargetsTasks/MonoTargetsTasks.csproj | 5 ++++ .../Templates/mono-bundled-assembly.template | 0 .../Templates/mono-bundled-data.template | 0 ...ce-preallocation-and-registration.template | 0 .../mono-bundled-satellite-assembly.template | 0 .../Templates/mono-bundled-symbol.template | 0 15 files changed, 25 insertions(+), 32 deletions(-) delete mode 100644 src/tasks/EmitBundleTask/EmitBundle.csproj rename src/tasks/{ => MonoTargetsTasks}/EmitBundleTask/EmitBundleBase.cs (97%) rename src/tasks/{ => MonoTargetsTasks}/EmitBundleTask/EmitBundleObjectFiles.cs (100%) rename src/tasks/{ => MonoTargetsTasks}/EmitBundleTask/EmitBundleSourceFiles.cs (100%) rename src/tasks/{EmitBundleTask => MonoTargetsTasks}/Templates/mono-bundled-assembly.template (100%) rename src/tasks/{EmitBundleTask => MonoTargetsTasks}/Templates/mono-bundled-data.template (100%) rename src/tasks/{EmitBundleTask => MonoTargetsTasks}/Templates/mono-bundled-resource-preallocation-and-registration.template (100%) rename src/tasks/{EmitBundleTask => MonoTargetsTasks}/Templates/mono-bundled-satellite-assembly.template (100%) rename src/tasks/{EmitBundleTask => MonoTargetsTasks}/Templates/mono-bundled-symbol.template (100%) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 91cb5f7b07e4fa..b78b6d272b75d6 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1518,8 +1518,11 @@ mono_assembly_open_from_bundle (MonoAssemblyLoadContext *alc, const char *filena MONO_ENTER_GC_UNSAFE; if (culture && culture [0] != 0) image = open_from_satellite_bundle (alc, filename, status, culture); - else - image = open_from_bundle_internal (alc, filename, status); + else { + char *name = g_path_get_basename (filename); + image = open_from_bundle_internal (alc, name, status); + g_free (name); + } if (image) { mono_image_addref (image); diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 53f819f2eb345d..06565b75e2189b 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -341,7 +341,7 @@ - + #include #include #include diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index ddeefca11d782d..1b183b83d9a10d 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -65,7 +65,7 @@ - + <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index 74316ac8c604e9..c58211278a7453 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -80,7 +80,7 @@ - + <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) diff --git a/src/tasks/EmitBundleTask/EmitBundle.csproj b/src/tasks/EmitBundleTask/EmitBundle.csproj deleted file mode 100644 index cfaa5eff214dc3..00000000000000 --- a/src/tasks/EmitBundleTask/EmitBundle.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - $(TargetFrameworkForNETCoreTasks) - enable - $(NoWarn),CA1050,CA1850 - - - - - - - - - - - - - - - - - - diff --git a/src/tasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs similarity index 97% rename from src/tasks/EmitBundleTask/EmitBundleBase.cs rename to src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index dd3084e6f75513..23d0092e82f366 100644 --- a/src/tasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -267,7 +267,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r { preloadedSymbolData = $",\n{Utils.GetEmbeddedResource("mono-bundled-symbol.template") .Replace("%ResourceSymbolName%", tuple.resourceSymbolName) - .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName].ToString())}"; + .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName!].ToString())}"; } preloadedStruct = preloadedStruct.Replace("%MonoBundledSymbolData%", preloadedSymbolData); preallocatedAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); @@ -351,8 +351,16 @@ private void BundleFileToCSource(string symbolName, bool shouldAddHeader, FileSt lock (symbolDataLen) { - if (!symbolDataLen.TryAdd(symbolName, generatedArrayLength) && symbolDataLen[symbolName] != generatedArrayLength) - Log.LogMessage(MessageImportance.High, $"There are duplicate resources with the same output symbol '{symbolName}' but have differing content sizes '{symbolDataLen[symbolName]}' != '{generatedArrayLength}'."); + int len = 0; + if (symbolDataLen.TryGetValue(symbolName, out len)) + { + if (len != generatedArrayLength) + Log.LogMessage(MessageImportance.High, $"There are duplicate resources with the same output symbol '{symbolName}' but have differing content sizes '{symbolDataLen[symbolName]}' != '{generatedArrayLength}'."); + } + else + { + symbolDataLen.Add(symbolName, generatedArrayLength); + } } } diff --git a/src/tasks/EmitBundleTask/EmitBundleObjectFiles.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs similarity index 100% rename from src/tasks/EmitBundleTask/EmitBundleObjectFiles.cs rename to src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs diff --git a/src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs similarity index 100% rename from src/tasks/EmitBundleTask/EmitBundleSourceFiles.cs rename to src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs diff --git a/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj b/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj index 181538f565e422..0912b76b05c63b 100644 --- a/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj +++ b/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj @@ -24,6 +24,11 @@ + + + + + diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-assembly.template similarity index 100% rename from src/tasks/EmitBundleTask/Templates/mono-bundled-assembly.template rename to src/tasks/MonoTargetsTasks/Templates/mono-bundled-assembly.template diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-data.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-data.template similarity index 100% rename from src/tasks/EmitBundleTask/Templates/mono-bundled-data.template rename to src/tasks/MonoTargetsTasks/Templates/mono-bundled-data.template diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template similarity index 100% rename from src/tasks/EmitBundleTask/Templates/mono-bundled-resource-preallocation-and-registration.template rename to src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-satellite-assembly.template similarity index 100% rename from src/tasks/EmitBundleTask/Templates/mono-bundled-satellite-assembly.template rename to src/tasks/MonoTargetsTasks/Templates/mono-bundled-satellite-assembly.template diff --git a/src/tasks/EmitBundleTask/Templates/mono-bundled-symbol.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-symbol.template similarity index 100% rename from src/tasks/EmitBundleTask/Templates/mono-bundled-symbol.template rename to src/tasks/MonoTargetsTasks/Templates/mono-bundled-symbol.template From 204cae7af513d03178bd5dd3bc1bc93e2e67d005 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Wed, 24 May 2023 00:19:59 -0400 Subject: [PATCH 081/110] WIP --- Directory.Build.props | 2 - src/mono/mono/metadata/assembly.c | 6 + .../apple/build/AppleBuild.LocalBuild.props | 3 - .../apple/build/AppleBuild.LocalBuild.targets | 1 - .../msbuild/common/LibraryBuilder.targets | 14 +- .../Sdk/Sdk.targets.in | 1 - .../Sdk/Sdk.targets.in | 1 - src/tasks/LibraryBuilder/LibraryBuilder.cs | 1 + .../EmitBundleTask/EmitBundleBase.cs | 185 +++++++++--------- .../EmitBundleTask/EmitBundleObjectFiles.cs | 4 + .../EmitBundleTask/EmitBundleSourceFiles.cs | 5 + 11 files changed, 111 insertions(+), 112 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index c072a65eb99a2d..cab8c8df24eb3e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -154,7 +154,6 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'LibraryBuilder', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoTargetsTasks', 'Debug', '$(NetCoreAppToolCurrent)')) - $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'EmitBundle', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'TestExclusionListTasks', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetFrameworkToolCurrent)', 'installer.tasks.dll')) @@ -169,7 +168,6 @@ $([MSBuild]::NormalizePath('$(LibraryBuilderDir)', 'LibraryBuilder.dll')) $([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll')) $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll')) - $([MSBuild]::NormalizePath('$(EmitBundleDir)', 'EmitBundle.dll')) $([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(RuntimeConfiguration)')) $(CoreCLRToolPath) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index b78b6d272b75d6..cdc352bbaae769 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1477,6 +1477,9 @@ bundled_assembly_match (const char *bundled_name, const char *name) static MonoImage * open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status) { + if (!mono_bundled_resources_contains_assemblies ()) + return NULL; + MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (filename); if (!assembly) return NULL; @@ -1487,6 +1490,9 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M static MonoImage * open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status, const char *culture) { + if (!mono_bundled_resources_contains_satellite_assemblies ()) + return NULL; + char *bundle_name = g_strconcat (culture, "/", filename, (const char *)NULL); MonoBundledSatelliteAssemblyResource *satellite_assembly = mono_bundled_resources_get_satellite_assembly_resource (bundle_name); diff --git a/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.props b/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.props index 7bd72e454b1e90..0c66b9186ee63e 100644 --- a/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.props +++ b/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.props @@ -37,7 +37,6 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'LibraryBuilder', 'Debug', '$(_NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(_NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoTargetsTasks', 'Debug', '$(_NetCoreAppToolCurrent)')) - $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'EmitBundle', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'mono', '$(TargetOS).$(TargetArchitecture).$(RuntimeConfig)')) $([MSBuild]::NormalizePath($(MonoArtifactsPath), 'cross', '$(TargetOS)-$(TargetArchitecture.ToLowerInvariant())')) @@ -55,7 +54,6 @@ $([MSBuild]::NormalizeDirectory($(BuildBaseDir), 'MonoTargetsTasks')) $([MSBuild]::NormalizeDirectory($(BuildBaseDir), 'AppleAppBuilder')) $([MSBuild]::NormalizeDirectory($(BuildBaseDir), 'LibraryBuilder')) - $([MSBuild]::NormalizeDirectory('$(BuildBaseDir)', 'EmitBundle')) $([MSBuild]::NormalizePath($(BuildBaseDir), 'cross')) <_MonoAotCrossCompilerPath>$([MSBuild]::NormalizePath($(MonoAotCrossDir), 'mono-aot-cross')) @@ -73,7 +71,6 @@ $([MSBuild]::NormalizePath('$(LibraryBuilderDir)', 'LibraryBuilder.dll')) $([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll')) $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll')) - $([MSBuild]::NormalizePath('$(EmitBundleDir)', 'EmitBundle.dll')) diff --git a/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.targets b/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.targets index 705d9a37a635aa..05bdce89bf8432 100644 --- a/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.targets +++ b/src/mono/msbuild/apple/build/AppleBuild.LocalBuild.targets @@ -62,7 +62,6 @@ $([MSBuild]::NormalizePath('$(LibraryBuilderDir)', 'LibraryBuilder.dll')) $([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll')) $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'RuntimeConfigParser.dll')) - $([MSBuild]::NormalizePath('$(EmitBundleDir)', 'EmitBundle.dll')) + AssemblyFile="$(MonoTargetsTasksAssemblyPath)" /> @@ -48,16 +48,10 @@ - mono-bundled-resource-data.c mono-bundled-source.c $(BundleDir) - - - - - <_ResourcesToBundle Remove="@(_ResourcesToBundle)" /> <_ResourcesToBundle Include="@(_AssembliesToBundleInternal)" /> @@ -74,9 +68,13 @@ + + + + + diff --git a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/Sdk.targets.in b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/Sdk.targets.in index 194fa2de4aecec..60d993f9c1a531 100644 --- a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/Sdk.targets.in +++ b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/Sdk.targets.in @@ -4,7 +4,6 @@ <_TasksDir Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETCoreTasks}\ <_TasksDir Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETFrameworkTasks}\ - $(_TasksDir)EmitBundle.dll $(_TasksDir)WasmAppBuilder.dll $(_TasksDir)WasmBuildTasks.dll $([MSBuild]::NormalizeDirectory($(MSBuildThisFileDirectory), '..', 'WasmAppHost')) diff --git a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/Sdk/Sdk.targets.in b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/Sdk/Sdk.targets.in index e53a67141e6da0..9fd83ffe152b55 100644 --- a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/Sdk/Sdk.targets.in +++ b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/Sdk/Sdk.targets.in @@ -4,7 +4,6 @@ <_TasksDir Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETCoreTasks}\ <_TasksDir Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETFrameworkTasks}\ - $(_TasksDir)EmitBundle.dll $(_TasksDir)WasmAppBuilder.dll $(_TasksDir)WasmBuildTasks.dll $([MSBuild]::NormalizeDirectory($(MSBuildThisFileDirectory), '..', 'WasmAppHost')) diff --git a/src/tasks/LibraryBuilder/LibraryBuilder.cs b/src/tasks/LibraryBuilder/LibraryBuilder.cs index b66d4838319588..e53a7f7c366478 100644 --- a/src/tasks/LibraryBuilder/LibraryBuilder.cs +++ b/src/tasks/LibraryBuilder/LibraryBuilder.cs @@ -307,6 +307,7 @@ private void WriteAutoInitializationFromTemplate() { throw new LogAsErrorException($"'{nameof(BundledRuntimeConfig)}' does not contain 'DataLenSymbol' metadata."); } + externBundledResourcesSymbols.AppendLine(); externBundledResourcesSymbols.AppendLine($"extern uint8_t {dataSymbol}[];"); externBundledResourcesSymbols.AppendLine($"extern const uint32_t {dataLenSymbol};"); } diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index 23d0092e82f366..8646c88af8b70b 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -17,6 +17,8 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab { private CancellationTokenSource BuildTaskCancelled { get; } = new(); + private Dictionary resourceDataSymbolDictionary = new(); + /// Must have DestinationFile metadata, which is the output filename /// Could have RegisteredName, otherwise it would be the filename. /// RegisteredName should be prefixed with namespace in form of unix like path. For example: "/usr/share/zoneinfo/" @@ -35,13 +37,6 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab /// public string? BundleFile { get; set; } - /// - /// Filename for the unified source containing all byte data and len - /// of the FilesToBundle resources. - /// Leave empty to keep sources separate. - /// - public string? CombinedResourceSource { get; set; } - /// /// Path to store build artifacts /// @@ -59,104 +54,98 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab public override bool Execute() { + List bundledResources = new(FilesToBundle.Length); + foreach (ITaskItem file in FilesToBundle) + { + ITaskItem bundledResource = file; + + var registeredName = file.GetMetadata("RegisteredName"); + if (string.IsNullOrEmpty(registeredName)) + { + registeredName = Path.GetFileName(file.ItemSpec); + file.SetMetadata("RegisteredName", registeredName); + } + + var destinationFile = file.GetMetadata("DestinationFile"); + if (string.IsNullOrEmpty(destinationFile)) + { + destinationFile = Path.GetFileNameWithoutExtension(file.ItemSpec) + GetDestinationFileExtension(); + file.SetMetadata("DestinationFile", destinationFile); + } + + string resourceDataSymbol = ToSafeSymbolName(destinationFile); + if (resourceDataSymbolDictionary.ContainsKey(registeredName)) + { + throw new LogAsErrorException($"Multiple resources have the same RegisteredName '{registeredName}'. Ensure {nameof(FilesToBundle)} 'RegisteredName' metadata are set and unique."); + } + resourceDataSymbolDictionary.Add(registeredName, resourceDataSymbol); + + bundledResources.Add(bundledResource); + } + // The DestinationFile (output filename) already includes a content hash. Grouping by this filename therefore // produces one group per file-content. We only want to emit one copy of each file-content, and one symbol for it. - var remainingDestinationFilesToBundle = FilesToBundle.GroupBy(f => f.GetMetadata("DestinationFile")).ToArray(); + var remainingDestinationFilesToBundle = bundledResources.GroupBy(file => file.GetMetadata("DestinationFile")).ToArray(); var verboseCount = 0; // Generate source file(s) containing each resource's byte data and size - if (string.IsNullOrEmpty(CombinedResourceSource)) + int allowedParallelism = Math.Max(Math.Min(bundledResources.Count, Environment.ProcessorCount), 1); + if (BuildEngine is IBuildEngine9 be9) + allowedParallelism = be9.RequestCores(allowedParallelism); + + Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => { - int allowedParallelism = Math.Max(Math.Min(remainingDestinationFilesToBundle.Length, Environment.ProcessorCount), 1); - if (BuildEngine is IBuildEngine9 be9) - allowedParallelism = be9.RequestCores(allowedParallelism); + var group = remainingDestinationFilesToBundle[i]; - Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => - { - var group = remainingDestinationFilesToBundle[i]; + var contentSourceFile = group.First(); - // Since the object filenames include a content hash, we can pick an arbitrary ITaskItem from each group, - // since we know each group's ITaskItems all contain the same binary data - var contentSourceFile = group.First(); + var inputFile = contentSourceFile.ItemSpec; + var destinationFile = contentSourceFile.GetMetadata("DestinationFile"); + var registeredName = contentSourceFile.GetMetadata("RegisteredName"); - var outputFile = group.Key; - var inputFile = contentSourceFile.ItemSpec; + var count = Interlocked.Increment(ref verboseCount); + Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); - var registeredName = contentSourceFile.GetMetadata("RegisteredName"); - if(string.IsNullOrEmpty(registeredName)) - { - registeredName = Path.GetFileName(inputFile); - } - var count = Interlocked.Increment(ref verboseCount); - Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); - - Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, outputFile); - var symbolName = ToSafeSymbolName(outputFile); - if (!Emit(outputFile, (codeStream) => { - using var inputStream = File.OpenRead(inputFile); - using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); - BundleFileToCSource(symbolName, true, inputStream, outputUtf8Writer); - })) - { - state.Stop(); - } - }); - } - else - { - Emit(Path.Combine(OutputDirectory, CombinedResourceSource), (codeStream) => { + Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, destinationFile); + var symbolName = resourceDataSymbolDictionary[registeredName]; + if (!Emit(destinationFile, (codeStream) => { + using var inputStream = File.OpenRead(inputFile); using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); - bool shouldAddHeader = true; - foreach (var destinationFileGroup in remainingDestinationFilesToBundle) - { - ITaskItem destinationFile = destinationFileGroup.First(); - var symbolName = ToSafeSymbolName(destinationFileGroup.Key); - using var inputStream = File.OpenRead(destinationFile.ItemSpec); - BundleFileToCSource(symbolName, shouldAddHeader, inputStream, outputUtf8Writer); - shouldAddHeader = false; - } - }); - } + BundleFileToCSource(symbolName, inputStream, outputUtf8Writer); + })) + { + state.Stop(); + } + }); - List bundledResources = new(FilesToBundle.Length); - foreach (ITaskItem file in FilesToBundle) + foreach (ITaskItem bundledResource in bundledResources) { - ITaskItem bundledResource = file; - string resourceDataSymbol = ToSafeSymbolName(bundledResource.GetMetadata("DestinationFile")); + string registeredName = bundledResource.GetMetadata("RegisteredName"); + string resourceDataSymbol = resourceDataSymbolDictionary[registeredName]; bundledResource.SetMetadata("DataSymbol", $"{resourceDataSymbol}_data"); bundledResource.SetMetadata("DataLenSymbol", $"{resourceDataSymbol}_data_len"); bundledResource.SetMetadata("DataLenSymbolValue", symbolDataLen[resourceDataSymbol].ToString()); - bundledResources.Add(bundledResource); } + BundledResources = bundledResources.ToArray(); if (!string.IsNullOrEmpty(BundleFile)) { - var filesToBundleByRegisteredName = FilesToBundle.GroupBy(file => { - var registeredName = file.GetMetadata("RegisteredName"); - if(string.IsNullOrEmpty(registeredName)) - { - registeredName = Path.GetFileName(file.ItemSpec); - } - return registeredName; - }).ToList(); - - string resourceSymbols = GatherUniqueExportedResourceDataSymbols(remainingDestinationFilesToBundle); + string resourceSymbols = GatherUniqueExportedResourceDataSymbols(bundledResources); - var files = filesToBundleByRegisteredName.Select(group => { - ITaskItem registeredFile = group.First(); - var registeredFilename = group.Key; - var resourceName = ToSafeSymbolName(registeredFilename, false); + var files = bundledResources.Select(bundledResource => { + var registeredName = bundledResource.GetMetadata("RegisteredName"); + var resourceName = ToSafeSymbolName(registeredName, false); // Different timezone resources may have the same contents, use registered name to differentiate preallocated resources - var resourceDataSymbol = ToSafeSymbolName(registeredFile.GetMetadata("DestinationFile")); + var resourceDataSymbol = resourceDataSymbolDictionary[registeredName]; - string culture = registeredFile.GetMetadata("Culture"); + string culture = bundledResource.GetMetadata("Culture"); string? resourceSymbolName = null; - if (File.Exists(registeredFile.GetMetadata("SymbolFile"))) - resourceSymbolName = ToSafeSymbolName(registeredFile.GetMetadata("SymbolFile")); + if (File.Exists(bundledResource.GetMetadata("SymbolFile"))) + resourceSymbolName = ToSafeSymbolName(bundledResource.GetMetadata("SymbolFile")); - return (registeredFilename, resourceName, resourceDataSymbol, culture, resourceSymbolName); + return (registeredName, resourceName, resourceDataSymbol, culture, resourceSymbolName); }).ToList(); Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); @@ -209,15 +198,18 @@ private static byte[] InitLookupTable() public abstract bool Emit(string destinationFile, Action inputProvider); + public abstract string GetDestinationFileExtension(); + private static Dictionary symbolDataLen = new(); - private static string GatherUniqueExportedResourceDataSymbols(IEnumerable> uniqueDestinationFiles) + private static string GatherUniqueExportedResourceDataSymbols(List uniqueDestinationFiles) { StringBuilder resourceSymbols = new (); HashSet resourcesAdded = new (); // Different Timezone resources may have the same contents foreach (var uniqueDestinationFile in uniqueDestinationFiles) { - string resourceDataSymbol = ToSafeSymbolName(uniqueDestinationFile.Key); + string destinationFile = uniqueDestinationFile.GetMetadata("DestinationFile"); + string resourceDataSymbol = ToSafeSymbolName(destinationFile); if (!resourcesAdded.Contains(resourceDataSymbol)) { resourceSymbols.AppendLine($"extern uint8_t {resourceDataSymbol}_data[];"); @@ -229,7 +221,7 @@ private static string GatherUniqueExportedResourceDataSymbols(IEnumerable files, StreamWriter outputUtf8Writer) + private static void GenerateBundledResourcePreallocationAndRegistration(string resourceSymbols, string bundleRegistrationFunctionName, ICollection<(string registeredName, string resourceName, string resourceDataSymbol, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) { List preallocatedSource = new (); @@ -246,7 +238,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r int dataCount = 0; foreach (var tuple in files) { - string resourceId = tuple.registeredFilename; + string resourceId = tuple.registeredName; // Generate Preloaded MonoBundled*Resource structs string preloadedStruct; @@ -254,11 +246,11 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r { preloadedStruct = satelliteAssemblyTemplate; preloadedStruct.Replace("%Culture%", tuple.culture); - resourceId = $"{tuple.culture}/{tuple.registeredFilename}"; + resourceId = $"{tuple.culture}/{tuple.registeredName}"; preallocatedSatelliteAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); satelliteAssembliesCount += 1; } - else if (tuple.registeredFilename.EndsWith(".dll")) + else if (tuple.registeredName.EndsWith(".dll")) { preloadedStruct = assemblyTemplate; // Add associated symfile information to MonoBundledAssemblyResource structs @@ -280,25 +272,27 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r dataCount += 1; } + var resourceDataSymbol = tuple.resourceDataSymbol; + preallocatedSource.Add(preloadedStruct.Replace("%ResourceName%", tuple.resourceName) - .Replace("%ResourceDataSymbol%", tuple.resourceDataSymbol) + .Replace("%ResourceDataSymbol%", resourceDataSymbol) .Replace("%ResourceID%", resourceId) - .Replace("%RegisteredFilename%", tuple.registeredFilename) - .Replace("%Len%", $"{tuple.resourceDataSymbol}_data_len_val")); + .Replace("%RegisteredFilename%", tuple.registeredName) + .Replace("%Len%", $"{resourceDataSymbol}_data_len_val")); } var addPreallocatedResources = new StringBuilder(); if (assembliesCount != 0) { - preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}bundledAssemblyResources[] = {{\n{string.Join(",\n", preallocatedAssemblies)}\n}};"); - addPreallocatedResources.AppendLine($" mono_bundled_resources_add ({bundleRegistrationFunctionName}bundledAssemblyResources, {assembliesCount});"); + preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}_assembly_resources[] = {{\n{string.Join(",\n", preallocatedAssemblies)}\n}};"); + addPreallocatedResources.AppendLine($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_assembly_resources, {assembliesCount});"); } if (satelliteAssembliesCount != 0) { - preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}bundledSatelliteAssemblyResources[] = {{\n{string.Join(",\n", preallocatedSatelliteAssemblies)}\n}};"); - addPreallocatedResources.AppendLine($" mono_bundled_resources_add ({bundleRegistrationFunctionName}bundledSatelliteAssemblyResources, {satelliteAssembliesCount});"); + preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}_satellite_assembly_resources[] = {{\n{string.Join(",\n", preallocatedSatelliteAssemblies)}\n}};"); + addPreallocatedResources.AppendLine($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_satellite_assembly_resources, {satelliteAssembliesCount});"); } if (dataCount != 0) { - preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}bundledDataResources[] = {{\n{string.Join(",\n", preallocatedData)}\n}};"); - addPreallocatedResources.AppendLine($" mono_bundled_resources_add ({bundleRegistrationFunctionName}bundledDataResources, {dataCount});"); + preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}_data_resources[] = {{\n{string.Join(",\n", preallocatedData)}\n}};"); + addPreallocatedResources.AppendLine($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_data_resources, {dataCount});"); } outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-resource-preallocation-and-registration.template") @@ -309,7 +303,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r .Replace("%AddPreallocatedResources%", addPreallocatedResources.ToString())); } - private void BundleFileToCSource(string symbolName, bool shouldAddHeader, FileStream inputStream, StreamWriter outputUtf8Writer) + private void BundleFileToCSource(string symbolName, FileStream inputStream, StreamWriter outputUtf8Writer) { // Emits a C source file in the same format as "xxd --include". Example: // @@ -323,8 +317,7 @@ private void BundleFileToCSource(string symbolName, bool shouldAddHeader, FileSt var generatedArrayLength = 0; var bytesEmitted = 0; - if (shouldAddHeader) - outputUtf8Writer.WriteLine("#include "); + outputUtf8Writer.WriteLine("#include "); outputUtf8Writer.Write($"uint8_t {symbolName}_data[] = {{"); outputUtf8Writer.Flush(); diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs index efdf11a23ec603..15f0deea840fd2 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs @@ -40,4 +40,8 @@ public override bool Emit(string destinationFile, Action inputProvider) return exitCode == 0; } + public override string GetDestinationFileExtension() + { + return ".o"; + } } diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs index 5e256c4eb66354..c67ba1c49f3b2f 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs @@ -19,4 +19,9 @@ public override bool Emit(string destinationFile, Action inputProvider) return true; } + + public override string GetDestinationFileExtension() + { + return ".c"; + } } From a77c39ce10b2485bd3f5a7360a78920b35f77430 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Wed, 24 May 2023 20:26:23 -0400 Subject: [PATCH 082/110] [mono] Alias assemblies with known extension in bundled resources hashtable --- src/mono/mono/metadata/assembly.c | 22 ++++----- src/mono/mono/metadata/bundled-resources.c | 52 +++++++++++++++++++++- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index cdc352bbaae769..3134fca28889b0 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1480,11 +1480,15 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M if (!mono_bundled_resources_contains_assemblies ()) return NULL; - MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (filename); + char *name = g_path_get_basename (filename); + MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (name); if (!assembly) return NULL; - return mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, assembly->resource.id, NULL); + MonoImage *image = mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, name, NULL); + + g_free (name); + return image; } static MonoImage * @@ -1494,13 +1498,14 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, return NULL; char *bundle_name = g_strconcat (culture, "/", filename, (const char *)NULL); - MonoBundledSatelliteAssemblyResource *satellite_assembly = mono_bundled_resources_get_satellite_assembly_resource (bundle_name); - g_free (bundle_name); if (!satellite_assembly) return NULL; - return mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, satellite_assembly->resource.id, NULL); + MonoImage *image = mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, bundle_name, NULL); + + g_free (bundle_name); + return image; } @@ -1524,11 +1529,8 @@ mono_assembly_open_from_bundle (MonoAssemblyLoadContext *alc, const char *filena MONO_ENTER_GC_UNSAFE; if (culture && culture [0] != 0) image = open_from_satellite_bundle (alc, filename, status, culture); - else { - char *name = g_path_get_basename (filename); - image = open_from_bundle_internal (alc, name, status); - g_free (name); - } + else + image = open_from_bundle_internal (alc, filename, status); if (image) { mono_image_addref (image); diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 5077a834c7cb9a..c65d8293af160b 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -6,6 +6,8 @@ #include #include +#include +#include static GHashTable *bundled_resources = NULL; static bool bundle_contains_assemblies = false; @@ -64,6 +66,54 @@ mono_bundled_resources_value_destroy_func (void *resource) value->free_bundled_resource_func (resource); } +static bool +is_known_assembly_extension (const char *ext) +{ + return !strcmp (ext, ".dll") || !strcmp (ext, ".webcil") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); +} + +static gboolean +resource_id_equal (const char *id_one, const char *id_two) +{ + const char *extension_one = strrchr (id_one, '.'); + const char *extension_two = strrchr (id_two, '.'); + if (extension_one && extension_two && is_known_assembly_extension (extension_one) && is_known_assembly_extension (extension_two)) { + size_t len_one = extension_one - id_one; + size_t len_two = extension_two - id_two; + return (len_one == len_two) && !strncmp (id_one, id_two, len_one); + } + + return !strcmp (id_one, id_two); +} + +static guint +resource_id_hash (const char *id) +{ + guint hash = 0; + size_t len = strlen (id); + char *extension = strrchr (id, '.'); + // alias all extensions to .dll + bool has_asm_extension = false; + if (extension && is_known_assembly_extension (extension)) { + len = extension - id; + has_asm_extension = true; + } + + char *p = (char *)id; + + while (p != id + len) + hash = (hash << 5) - (hash + (unsigned char)*p++); + + if (has_asm_extension) { + hash = (hash << 5) - (hash + '.'); + hash = (hash << 5) - (hash + 'd'); + hash = (hash << 5) - (hash + 'l'); + hash = (hash << 5) - (hash + 'l'); + } + + return hash; +} + //--------------------------------------------------------------------------------------- // // mono_bundled_resources_add handles bundling of many types of resources to circumvent @@ -87,7 +137,7 @@ void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len) { if (!bundled_resources) - bundled_resources = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, mono_bundled_resources_value_destroy_func); + bundled_resources = g_hash_table_new_full ((GHashFunc)resource_id_hash, (GEqualFunc)resource_id_equal, NULL, mono_bundled_resources_value_destroy_func); bool assemblyAdded = false; bool satelliteAssemblyAdded = false; From ec2570d517320c990db625d58a990da450ac4aa7 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 25 May 2023 00:42:43 -0400 Subject: [PATCH 083/110] [tasks] Calculate destination files within EmitBundle task itself --- .../msbuild/common/LibraryBuilder.targets | 8 +---- .../EmitBundleTask/EmitBundleBase.cs | 36 ++++++++++++------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index d2270cf952896a..f76fbb3331d53e 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -20,6 +20,7 @@ <_ExtraLibrarySources Include="$(_AotModuleTablePath)" /> <_ExtraLibrarySources Include="@(BundledSources)" /> + <_ExtraLibrarySources Include="%(_BundledResources.DestinationFile)" /> <_ExtraLinkerArgs Include="@(_CommonLinkerArgs)" /> <_BundledRuntimeConfig Include="@(_BundledResources)" Condition="$([System.String]::Copy('%(Identity)').EndsWith('runtimeconfig.bin'))" /> @@ -62,15 +63,8 @@ - - <_ResourcesToBundleWithHashes Update="@(_ResourcesToBundleWithHashes)"> - $(BundleOutputDirectory)%(_ResourcesToBundleWithHashes.Filename)%(_ResourcesToBundleWithHashes.Extension).$([System.String]::Copy(%(_ResourcesToBundleWithHashes.FileHash)).Substring(0, 8)).c - - - - resourceDataSymbolDictionary = new(); + private Dictionary resourcesForDataSymbolDictionary= new(); + /// Must have DestinationFile metadata, which is the output filename /// Could have RegisteredName, otherwise it would be the filename. /// RegisteredName should be prefixed with namespace in form of unix like path. For example: "/usr/share/zoneinfo/" @@ -66,20 +68,28 @@ public override bool Execute() file.SetMetadata("RegisteredName", registeredName); } - var destinationFile = file.GetMetadata("DestinationFile"); - if (string.IsNullOrEmpty(destinationFile)) - { - destinationFile = Path.GetFileNameWithoutExtension(file.ItemSpec) + GetDestinationFileExtension(); - file.SetMetadata("DestinationFile", destinationFile); - } - - string resourceDataSymbol = ToSafeSymbolName(destinationFile); + string resourceDataSymbol = $"bundled_resource_{ToSafeSymbolName(Utils.ComputeHash(file.ItemSpec))}"; if (resourceDataSymbolDictionary.ContainsKey(registeredName)) { throw new LogAsErrorException($"Multiple resources have the same RegisteredName '{registeredName}'. Ensure {nameof(FilesToBundle)} 'RegisteredName' metadata are set and unique."); } + Log.LogMessage(MessageImportance.High, $"Adding '{registeredName}':'{resourceDataSymbol}'"); resourceDataSymbolDictionary.Add(registeredName, resourceDataSymbol); + file.SetMetadata("DestinationFile", resourceDataSymbol + GetDestinationFileExtension()); + + string[] resourcesWithDataSymbol; + if (resourcesForDataSymbolDictionary.TryGetValue(resourceDataSymbol, out string[]? resourcesAlreadyWithDataSymbol)) + { + resourcesForDataSymbolDictionary.Remove(resourceDataSymbol); + resourcesWithDataSymbol = resourcesAlreadyWithDataSymbol.Append(registeredName).ToArray(); + } + else + { + resourcesWithDataSymbol = new[] {registeredName}; + } + resourcesForDataSymbolDictionary.Add(resourceDataSymbol, resourcesWithDataSymbol); + bundledResources.Add(bundledResource); } @@ -109,7 +119,7 @@ public override bool Execute() Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, destinationFile); var symbolName = resourceDataSymbolDictionary[registeredName]; - if (!Emit(destinationFile, (codeStream) => { + if (!Emit(Path.Combine(OutputDirectory, destinationFile), (codeStream) => { using var inputStream = File.OpenRead(inputFile); using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); BundleFileToCSource(symbolName, inputStream, outputUtf8Writer); @@ -202,14 +212,14 @@ private static byte[] InitLookupTable() private static Dictionary symbolDataLen = new(); - private static string GatherUniqueExportedResourceDataSymbols(List uniqueDestinationFiles) + private string GatherUniqueExportedResourceDataSymbols(List uniqueDestinationFiles) { StringBuilder resourceSymbols = new (); HashSet resourcesAdded = new (); // Different Timezone resources may have the same contents foreach (var uniqueDestinationFile in uniqueDestinationFiles) { - string destinationFile = uniqueDestinationFile.GetMetadata("DestinationFile"); - string resourceDataSymbol = ToSafeSymbolName(destinationFile); + string registeredName = uniqueDestinationFile.GetMetadata("RegisteredName"); + string resourceDataSymbol = resourceDataSymbolDictionary[registeredName]; if (!resourcesAdded.Contains(resourceDataSymbol)) { resourceSymbols.AppendLine($"extern uint8_t {resourceDataSymbol}_data[];"); @@ -319,6 +329,8 @@ private void BundleFileToCSource(string symbolName, FileStream inputStream, Stre outputUtf8Writer.WriteLine("#include "); + string[] resourcesForDataSymbol = resourcesForDataSymbolDictionary[symbolName]; + outputUtf8Writer.WriteLine($"// Resource Registered Names: {string.Join(", ", resourcesForDataSymbol)}"); outputUtf8Writer.Write($"uint8_t {symbolName}_data[] = {{"); outputUtf8Writer.Flush(); while ((bytesRead = inputStream.Read(buf, 0, buf.Length)) > 0) From 4ba29ad0c07bea3d2061623d21c971ff92bb888c Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 25 May 2023 01:45:12 -0400 Subject: [PATCH 084/110] Update invocations of EmitBundle to grab DestinationFiles from output --- src/mono/wasi/build/WasiApp.Native.targets | 26 +++++++++------------- src/mono/wasi/wasi.proj | 10 ++++----- src/mono/wasm/wasm.proj | 16 ++++++------- 3 files changed, 22 insertions(+), 30 deletions(-) diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 06565b75e2189b..035b7518b8b1ed 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -358,33 +358,26 @@ - - - $(_WasmIntermediateOutputPath)%(WasmBundleAssembliesWithHashes.Filename)%(WasmBundleAssembliesWithHashes.Extension).$([System.String]::Copy(%(WasmBundleAssembliesWithHashes.FileHash)).Substring(0, 8)).o - - - $(_WasmIntermediateOutputPath)%(WasmBundleIcuWithHashes.Filename)%(WasmBundleIcuWithHashes.Extension).$([System.String]::Copy(%(WasmBundleIcuWithHashes.FileHash)).Substring(0, 8)).o - - + OutputDirectory="$(_WasmIntermediateOutputPath)"> + + <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmAssembliesBundleObjectFile)" /> - <_WasiObjectFilesForBundle Include="%(WasmBundleAssembliesWithHashes.DestinationFile)" /> + <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)%(BundledWasmAssemblies.DestinationFile)" /> - + + OutputDirectory="$(_WasmIntermediateOutputPath)"> + + <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmIcuBundleObjectFile)" /> - <_WasiObjectFilesForBundle Include="%(WasmBundleIcuWithHashes.DestinationFile)" /> + <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)%(BundledWasmIcu.DestinationFile)" /> - + diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index 1b183b83d9a10d..4c83d10580679e 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -81,7 +81,6 @@ <_WasmBundleTimezonesWithHashes Update="@(_WasmBundleTimezonesWithHashes)"> - $(WasiObjDir)\wasm-bundled-$([System.String]::Copy(%(_WasmBundleTimezonesWithHashes.FileHash))).o /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmBundleTimezonesWithHashes.Identity)).Replace('\','/')) @@ -91,10 +90,11 @@ ClangExecutable="$(WasiClang)" BundleRegistrationFunctionName="mono_register_timezones_bundle" BundleFile="$(_WasmTimezonesBundleObjectFile)" - OutputDirectory="$(WasiObjDir)" - /> + OutputDirectory="$(WasiObjDir)"> + + - + <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> <_WasmBundleTimezonesToDelete Remove="$(WasiObjDir)\$(_WasmTimezonesBundleObjectFile)" /> - <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(WasiObjDir)\%(BundledWasiTimezones.DestinationFile)" /> diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index c58211278a7453..326de3be7146d3 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -98,8 +98,6 @@ <_WasmBundleTimezonesWithHashes Update="@(_WasmBundleTimezonesWithHashes)"> - $(WasmObjDir)\wasm-bundled-$([System.String]::Copy(%(_WasmBundleTimezonesWithHashes.FileHash))).c - $(WasmObjDir)\wasm-bundled-$([System.String]::Copy(%(_WasmBundleTimezonesWithHashes.FileHash))).o /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmBundleTimezonesWithHashes.Identity)).Replace('\','/')) @@ -108,11 +106,12 @@ FilesToBundle="@(_WasmBundleTimezonesWithHashes)" BundleRegistrationFunctionName="mono_register_timezones_bundle" BundleFile="$(_WasmTimezonesBundleSourceFile)" - OutputDirectory="$(WasmObjDir)" - /> + OutputDirectory="$(WasmObjDir)"> + + - <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), %(_WasmBundleTimezonesWithHashes.DestinationFile)).Replace('\','/'))" /> + <_WasmBundleTimezonesSources Include="%(BundledWasmTimezones.DestinationFile)" /> <_WasmBundleTimezonesSources Include="$(_WasmTimezonesBundleSourceFile)" /> - - + <_WasmBundleTimezonesToDelete Remove="$(WasmObjDir)\$(_WasmTimezonesBundleSourceFile)" /> <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> - <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.DestinationFile)" /> - <_WasmBundleTimezonesToDelete Remove="%(_WasmBundleTimezonesWithHashes.ObjectFile)" /> + <_WasmBundleTimezonesToDelete Remove="$(WasmObjDir)\%(BundledWasmTimezones.DestinationFile)" /> + <_WasmBundleTimezonesToDelete Remove="%(WasmBundleTimezonesObjects)" /> From 9e5e42f26a76842e54ab7a1697dcbd7758ce5384 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 25 May 2023 12:36:08 -0400 Subject: [PATCH 085/110] Guard webcil specific logic --- src/mono/mono/metadata/assembly.c | 26 --------------- src/mono/mono/metadata/bundled-resources.c | 37 +++++++++++++--------- 2 files changed, 22 insertions(+), 41 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 3134fca28889b0..90705a7551212c 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1448,32 +1448,6 @@ absolute_dir (const gchar *filename) return res; } -static gboolean -bundled_assembly_match (const char *bundled_name, const char *name) -{ -#ifndef ENABLE_WEBCIL - return strcmp (bundled_name, name) == 0; -#else - if (strcmp (bundled_name, name) == 0) - return TRUE; - /* if they want a .dll and we have the matching .webcil, return it */ - if (g_str_has_suffix (bundled_name, ".webcil") && g_str_has_suffix (name, ".dll")) { - size_t bprefix = strlen (bundled_name) - strlen (".webcil"); - size_t nprefix = strlen (name) - strlen (".dll"); - if (bprefix == nprefix && strncmp (bundled_name, name, bprefix) == 0) - return TRUE; - } - /* if they want a .dll and we have the matching .wasm webcil-in-wasm, return it */ - if (g_str_has_suffix (bundled_name, MONO_WEBCIL_IN_WASM_EXTENSION) && g_str_has_suffix (name, ".dll")) { - size_t bprefix = strlen (bundled_name) - strlen (MONO_WEBCIL_IN_WASM_EXTENSION); - size_t nprefix = strlen (name) - strlen (".dll"); - if (bprefix == nprefix && strncmp (bundled_name, name, bprefix) == 0) - return TRUE; - } - return FALSE; -#endif -} - static MonoImage * open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, MonoImageOpenStatus *status) { diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index c65d8293af160b..b7b5d415affd64 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -66,6 +66,7 @@ mono_bundled_resources_value_destroy_func (void *resource) value->free_bundled_resource_func (resource); } +#ifdef ENABLE_WEBCIL static bool is_known_assembly_extension (const char *ext) { @@ -89,23 +90,23 @@ resource_id_equal (const char *id_one, const char *id_two) static guint resource_id_hash (const char *id) { + const char *current = id; + const char *extension = NULL; + guint previous_hash = 0; guint hash = 0; - size_t len = strlen (id); - char *extension = strrchr (id, '.'); - // alias all extensions to .dll - bool has_asm_extension = false; - if (extension && is_known_assembly_extension (extension)) { - len = extension - id; - has_asm_extension = true; - } - char *p = (char *)id; - - while (p != id + len) - hash = (hash << 5) - (hash + (unsigned char)*p++); + while (*current) { + hash = (hash << 5) - (hash + *current); + if (*current == '.') { + extension = current; + previous_hash = hash; + } + current++; + } - if (has_asm_extension) { - hash = (hash << 5) - (hash + '.'); + // alias all extensions to .dll + if (extension && is_known_assembly_extension (extension)) { + hash = previous_hash; hash = (hash << 5) - (hash + 'd'); hash = (hash << 5) - (hash + 'l'); hash = (hash << 5) - (hash + 'l'); @@ -113,6 +114,7 @@ resource_id_hash (const char *id) return hash; } +#endif // ENABLE_WEBCIL //--------------------------------------------------------------------------------------- // @@ -136,8 +138,13 @@ resource_id_hash (const char *id) void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len) { - if (!bundled_resources) + if (!bundled_resources) { +#ifdef ENABLE_WEBCIL bundled_resources = g_hash_table_new_full ((GHashFunc)resource_id_hash, (GEqualFunc)resource_id_equal, NULL, mono_bundled_resources_value_destroy_func); +#else + bundled_resources = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, mono_bundled_resources_value_destroy_func); +#endif + } bool assemblyAdded = false; bool satelliteAssemblyAdded = false; From d0fafb6d85b07c97dc526cf08ef285387ea0b4c3 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 25 May 2023 12:43:37 -0400 Subject: [PATCH 086/110] Include OutputDirectory path into output destinationFiles --- src/mono/wasi/build/WasiApp.Native.targets | 8 ++++---- src/mono/wasi/wasi.proj | 4 ++-- src/mono/wasm/wasm.proj | 4 ++-- .../MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs | 5 ++--- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 035b7518b8b1ed..98c9be7b4f4985 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -370,14 +370,14 @@ <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmAssembliesBundleObjectFile)" /> - <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)%(BundledWasmAssemblies.DestinationFile)" /> + <_WasiObjectFilesForBundle Include="%(BundledWasmAssemblies.DestinationFile)" /> - + <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmIcuBundleObjectFile)" /> - <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)%(BundledWasmIcu.DestinationFile)" /> + <_WasiObjectFilesForBundle Include="%(BundledWasmIcu.DestinationFile)" /> - + diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index 4c83d10580679e..7c3d4b5469ae7a 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -94,7 +94,7 @@ - + <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> <_WasmBundleTimezonesToDelete Remove="$(WasiObjDir)\$(_WasmTimezonesBundleObjectFile)" /> - <_WasmBundleTimezonesToDelete Remove="$(WasiObjDir)\%(BundledWasiTimezones.DestinationFile)" /> + <_WasmBundleTimezonesToDelete Remove="%(BundledWasiTimezones.DestinationFile)" /> diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index 326de3be7146d3..2aff5404a4322c 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -111,7 +111,7 @@ - <_WasmBundleTimezonesSources Include="%(BundledWasmTimezones.DestinationFile)" /> + <_WasmBundleTimezonesSources Include="$([MSBuild]::MakeRelative($(WasmObjDir), %(BundledWasmTimezones.DestinationFile)).Replace('\','/'))" /> <_WasmBundleTimezonesSources Include="$(_WasmTimezonesBundleSourceFile)" /> <_WasmBundleTimezonesToDelete Remove="$(WasmObjDir)\$(_WasmTimezonesBundleSourceFile)" /> <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> - <_WasmBundleTimezonesToDelete Remove="$(WasmObjDir)\%(BundledWasmTimezones.DestinationFile)" /> + <_WasmBundleTimezonesToDelete Remove="%(BundledWasmTimezones.DestinationFile)" /> <_WasmBundleTimezonesToDelete Remove="%(WasmBundleTimezonesObjects)" /> diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index 44e6871c051a92..0b2e969b86fc9e 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -73,10 +73,9 @@ public override bool Execute() { throw new LogAsErrorException($"Multiple resources have the same RegisteredName '{registeredName}'. Ensure {nameof(FilesToBundle)} 'RegisteredName' metadata are set and unique."); } - Log.LogMessage(MessageImportance.High, $"Adding '{registeredName}':'{resourceDataSymbol}'"); resourceDataSymbolDictionary.Add(registeredName, resourceDataSymbol); - file.SetMetadata("DestinationFile", resourceDataSymbol + GetDestinationFileExtension()); + file.SetMetadata("DestinationFile", Path.Combine(OutputDirectory, resourceDataSymbol + GetDestinationFileExtension())); string[] resourcesWithDataSymbol; if (resourcesForDataSymbolDictionary.TryGetValue(resourceDataSymbol, out string[]? resourcesAlreadyWithDataSymbol)) @@ -119,7 +118,7 @@ public override bool Execute() Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, destinationFile); var symbolName = resourceDataSymbolDictionary[registeredName]; - if (!Emit(Path.Combine(OutputDirectory, destinationFile), (codeStream) => { + if (!Emit(destinationFile, (codeStream) => { using var inputStream = File.OpenRead(inputFile); using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); BundleFileToCSource(symbolName, inputStream, outputUtf8Writer); From 5dad7112f40eef59e14f6a7fff22db11a14fbbb5 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 25 May 2023 14:50:54 -0400 Subject: [PATCH 087/110] Fix debugger tests and memory leak --- src/mono/mono/component/mini-wasm-debugger.c | 8 +++++--- src/mono/mono/metadata/assembly.c | 16 ++++++++-------- src/mono/mono/mini/mini-wasm.h | 1 - src/mono/wasm/runtime/driver.c | 18 ------------------ 4 files changed, 13 insertions(+), 30 deletions(-) diff --git a/src/mono/mono/component/mini-wasm-debugger.c b/src/mono/mono/component/mini-wasm-debugger.c index 2b38e5c17bbbb6..9a550eee6ba2d7 100644 --- a/src/mono/mono/component/mini-wasm-debugger.c +++ b/src/mono/mono/component/mini-wasm-debugger.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -436,9 +437,10 @@ mono_wasm_send_dbg_command (int id, MdbgProtCommandSet command_set, int command, else { unsigned int assembly_size = 0; - int symfile_size = 0; - const unsigned char* assembly_bytes = mono_wasm_get_assembly_bytes (assembly_name, &assembly_size); - const unsigned char* pdb_bytes = mono_get_symfile_bytes_from_bundle (assembly_name, &symfile_size); + unsigned int symfile_size = 0; + const unsigned char* assembly_bytes = NULL; + const unsigned char* pdb_bytes = NULL; + mono_bundled_resources_get_assembly_resource_values (assembly_name, &assembly_bytes, &assembly_size, &pdb_bytes, &symfile_size); m_dbgprot_buffer_init (&buf, assembly_size + symfile_size); m_dbgprot_buffer_add_byte_array (&buf, (uint8_t *) assembly_bytes, assembly_size); m_dbgprot_buffer_add_byte_array (&buf, (uint8_t *) pdb_bytes, symfile_size); diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 90705a7551212c..58261eeade8763 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1454,12 +1454,12 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M if (!mono_bundled_resources_contains_assemblies ()) return NULL; + MonoImage *image = NULL; char *name = g_path_get_basename (filename); - MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (name); - if (!assembly) - return NULL; - MonoImage *image = mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, name, NULL); + MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (name); + if (assembly) + image = mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, name, NULL); g_free (name); return image; @@ -1471,12 +1471,12 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, if (!mono_bundled_resources_contains_satellite_assemblies ()) return NULL; + MonoImage *image = NULL; char *bundle_name = g_strconcat (culture, "/", filename, (const char *)NULL); - MonoBundledSatelliteAssemblyResource *satellite_assembly = mono_bundled_resources_get_satellite_assembly_resource (bundle_name); - if (!satellite_assembly) - return NULL; - MonoImage *image = mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, bundle_name, NULL); + MonoBundledSatelliteAssemblyResource *satellite_assembly = mono_bundled_resources_get_satellite_assembly_resource (bundle_name); + if (satellite_assembly) + image = mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, bundle_name, NULL); g_free (bundle_name); return image; diff --git a/src/mono/mono/mini/mini-wasm.h b/src/mono/mono/mini/mini-wasm.h index 61e81e8a81fbdb..365c178903f15f 100644 --- a/src/mono/mono/mini/mini-wasm.h +++ b/src/mono/mono/mini/mini-wasm.h @@ -99,7 +99,6 @@ G_EXTERN_C void mono_wasm_enable_debugging (int log_level); void mono_wasm_set_timeout (int timeout); int mono_wasm_assembly_already_added (const char *assembly_name); -const unsigned char *mono_wasm_get_assembly_bytes (const char *name, unsigned int *size); void mono_wasm_print_stack_trace (void); diff --git a/src/mono/wasm/runtime/driver.c b/src/mono/wasm/runtime/driver.c index edb23669e40499..91c6186f6e201c 100644 --- a/src/mono/wasm/runtime/driver.c +++ b/src/mono/wasm/runtime/driver.c @@ -228,24 +228,6 @@ mono_wasm_assembly_already_added (const char *assembly_name) return 0; } -const unsigned char * -mono_wasm_get_assembly_bytes (const char *assembly_name, unsigned int *size) -{ - if (assembly_count == 0) - return 0; - - WasmAssembly *entry = assemblies; - while (entry != NULL) { - if (strcmp (entry->assembly.name, assembly_name) == 0) - { - *size = entry->assembly.size; - return entry->assembly.data; - } - entry = entry->next; - } - return NULL; -} - typedef struct WasmSatelliteAssembly_ WasmSatelliteAssembly; struct WasmSatelliteAssembly_ { From 51637a356d193b1330fdf145c6d38178cd106ce6 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 25 May 2023 16:56:44 -0400 Subject: [PATCH 088/110] Various fixups and Documentation --- src/mono/mono/metadata/assembly.c | 6 +- src/mono/mono/metadata/bundled-resources.c | 93 ++++++++++++++++++- src/mono/mono/metadata/mono-debug.c | 3 + .../msbuild/common/LibraryBuilder.targets | 6 +- src/mono/wasi/build/WasiApp.Native.targets | 10 +- src/mono/wasi/wasi.proj | 13 +-- src/mono/wasm/wasm.proj | 2 - .../EmitBundleTask/EmitBundleBase.cs | 13 +-- ...ce-preallocation-and-registration.template | 3 +- 9 files changed, 113 insertions(+), 36 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 58261eeade8763..e011d2a0899a02 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1480,7 +1480,6 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, g_free (bundle_name); return image; - } /** @@ -3134,12 +3133,15 @@ mono_assembly_get_name_internal (MonoAssembly *assembly) /** * mono_register_bundled_assemblies: + * Dynamically allocates MonoBundledAssemblyResources to leverage + * preferred bundling api mono_bundled_resources_add. */ void mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) { for (int i = 0; assemblies [i]; ++i) { const MonoBundledAssembly *assembly = assemblies [i]; + // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (assembly->name); if (!assembly_resource) { assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); @@ -3175,6 +3177,8 @@ mono_create_new_bundled_satellite_assembly (const char *name, const char *cultur /** * mono_register_bundled_satellite_assemblies: + * Dynamically allocates MonoBundledSatelliteAssemblyResources to leverage + * preferred bundling api mono_bundled_resources_add. */ void mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly **satellite_assemblies) diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index b7b5d415affd64..bf9397ab5f8a7b 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -170,9 +170,7 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t //--------------------------------------------------------------------------------------- // // mono_bundled_resources_get retrieves the pointer of the MonoBundledResource associated -// with a key equivalent to the requested resource id. If the requested bundled resource's -// name has been added via mono_bundled_resources_add, a MonoBundled*Resource had been -// preallocated, typically through EmitBundleTask. +// with a key equivalent to the requested resource id. // // Arguments: // * id - Unique name of the resource @@ -190,6 +188,21 @@ mono_bundled_resources_get (const char *id) return g_hash_table_lookup (bundled_resources, id); } +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_assembly_resource retrieves MonoBundledAssemblyResource* associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// +// Returns: +// MonoBundledAssemblyResource * - Pointer to the bundled assembly resource in the hashmap with the key `id` +// +// Note: As MonoBundled*Resource types are not public, prefer `mono_bundled_resources_get_assembly_resource_values` +// in external contexts to grab assembly and symbol data. +// + MonoBundledAssemblyResource * mono_bundled_resources_get_assembly_resource (const char *id) { @@ -201,6 +214,21 @@ mono_bundled_resources_get_assembly_resource (const char *id) return assembly; } +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_satellite_assembly_resource retrieves MonoBundledSatelliteAssemblyResource* associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// +// Returns: +// MonoBundledSatelliteAssemblyResource * - Pointer to the bundled assembly resource in the hashmap with the key `id` +// +// Note: As MonoBundled*Resource types are not public, prefer `mono_bundled_resources_get_satellite_assembly_resource_values` +// in external contexts to grab satellite assembly data. +// + MonoBundledSatelliteAssemblyResource * mono_bundled_resources_get_satellite_assembly_resource (const char *id) { @@ -212,6 +240,21 @@ mono_bundled_resources_get_satellite_assembly_resource (const char *id) return satellite_assembly; } +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_data_resource retrieves MonoBundledDataResource* associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// +// Returns: +// MonoBundledDataResource * - Pointer to the bundled assembly resource in the hashmap with the key `id` +// +// Note: As MonoBundled*Resource types are not public, prefer `mono_bundled_resources_get_data_resource_values` +// in external contexts to grab data. +// + MonoBundledDataResource * mono_bundled_resources_get_data_resource (const char *id) { @@ -223,6 +266,22 @@ mono_bundled_resources_get_data_resource (const char *id) return data; } +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_assembly_resource_values retrieves assembly data associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// ** data_out - address to point at assembly byte data +// ** size_out - address to point at assembly byte data size +// ** symbol_data_out - address to point at assembly symbol byte data +// ** symbol_size_out - address to point at assembly symbol byte data size +// +// Returns: +// bool - whether or not a valid MonoBundledAssemblyResource was found with key 'id' +// + bool mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out, const uint8_t **symbol_data_out, uint32_t *symbol_size_out) { @@ -244,6 +303,20 @@ mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8 return true; } +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_satellite_assembly_resource_values retrieves satellite assembly data associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// ** data_out - address to point at satellite assembly byte data +// ** size_out - address to point at satellite assembly byte data size +// +// Returns: +// bool - whether or not a valid MonoBundledSatelliteAssemblyResource was found with key 'id' +// + bool mono_bundled_resources_get_satellite_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out) { @@ -262,6 +335,20 @@ mono_bundled_resources_get_satellite_assembly_resource_values (const char *id, c return true; } +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_data_resource_values retrieves data associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// ** data_out - address to point at resource byte data +// ** size_out - address to point at resource byte data size +// +// Returns: +// bool - whether or not a valid MonoBundledDataResource was found with key 'id' +// + bool mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out) { diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 1b62f9b5bf9be5..a57ca41b35c3ae 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1070,10 +1070,13 @@ mono_is_debugger_attached (void) /** * mono_register_symfile_for_assembly: + * Dynamically allocates MonoBundledAssemblyResource to leverage + * preferred bundling api mono_bundled_resources_add. */ void mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte *raw_contents, int size) { + // Check if assembly dll counterpart had been added via mono_register_bundled_assemblies MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (assembly_name); if (!assembly_resource) { assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); diff --git a/src/mono/msbuild/common/LibraryBuilder.targets b/src/mono/msbuild/common/LibraryBuilder.targets index f76fbb3331d53e..9ea4a4562ee1e9 100644 --- a/src/mono/msbuild/common/LibraryBuilder.targets +++ b/src/mono/msbuild/common/LibraryBuilder.targets @@ -59,16 +59,12 @@ <_ResourcesToBundle Include="$(_ParsedRuntimeConfigFilePath)" /> - - - - diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 98c9be7b4f4985..7108e7841ab483 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -352,15 +352,9 @@ <_WasmAssembliesBundleObjectFile>wasi_bundled_assemblies.o <_WasmIcuBundleObjectFile>wasi_bundled_icu.o - - - - - - <_WasmTimezonesInternal Include="$(_WasmTimezonesPath)\**\*.*" WasmRole="Timezone"/> - - - - <_WasmBundleTimezonesWithHashes Update="@(_WasmBundleTimezonesWithHashes)"> - /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmBundleTimezonesWithHashes.Identity)).Replace('\','/')) - + <_WasmTimezonesInternal Update="@(_WasmTimezonesInternal)"> + /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmTimezonesInternal.Identity)).Replace('\','/')) + - + <_WasmTimezonesPath>$([MSBuild]::NormalizePath('$(PkgSystem_Runtime_TimeZoneData)', 'contentFiles', 'any', 'any', 'data')) <_WasmTimezonesBundleSourceFile>wasm-bundled-timezones.c - <_WasmTimezonesBundleObjectFile>$(WasmObjDir)\wasm-bundled-timezones.o <_WasmTimezonesBundleArchive>$(WasmObjDir)\wasm-bundled-timezones.a <_WasmTimezonesSourcesRsp>$(WasmObjDir)\wasm-bundled-timezones-sources.rsp <_WasmTimezonesArchiveRsp>$(WasmObjDir)\wasm-bundled-timezones-archive.rsp @@ -147,7 +146,6 @@ <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.o" /> <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.c" /> <_WasmBundleTimezonesToDelete Remove="$(WasmObjDir)\$(_WasmTimezonesBundleSourceFile)" /> - <_WasmBundleTimezonesToDelete Remove="$(_WasmTimezonesBundleObjectFile)" /> <_WasmBundleTimezonesToDelete Remove="%(BundledWasmTimezones.DestinationFile)" /> <_WasmBundleTimezonesToDelete Remove="%(WasmBundleTimezonesObjects)" /> diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index 0b2e969b86fc9e..24152a88792bf2 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -290,18 +290,18 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r .Replace("%Len%", $"{resourceDataSymbol}_data_len_val")); } - var addPreallocatedResources = new StringBuilder(); + List addPreallocatedResources = new (); if (assembliesCount != 0) { preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}_assembly_resources[] = {{\n{string.Join(",\n", preallocatedAssemblies)}\n}};"); - addPreallocatedResources.AppendLine($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_assembly_resources, {assembliesCount});"); + addPreallocatedResources.Add($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_assembly_resources, {assembliesCount});"); } if (satelliteAssembliesCount != 0) { preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}_satellite_assembly_resources[] = {{\n{string.Join(",\n", preallocatedSatelliteAssemblies)}\n}};"); - addPreallocatedResources.AppendLine($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_satellite_assembly_resources, {satelliteAssembliesCount});"); + addPreallocatedResources.Add($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_satellite_assembly_resources, {satelliteAssembliesCount});"); } if (dataCount != 0) { preallocatedResources.AppendLine($"MonoBundledResource *{bundleRegistrationFunctionName}_data_resources[] = {{\n{string.Join(",\n", preallocatedData)}\n}};"); - addPreallocatedResources.AppendLine($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_data_resources, {dataCount});"); + addPreallocatedResources.Add($" mono_bundled_resources_add ({bundleRegistrationFunctionName}_data_resources, {dataCount});"); } outputUtf8Writer.Write(Utils.GetEmbeddedResource("mono-bundled-resource-preallocation-and-registration.template") @@ -309,7 +309,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r .Replace("%PreallocatedStructs%", string.Join("\n", preallocatedSource)) .Replace("%PreallocatedResources%", preallocatedResources.ToString()) .Replace("%BundleRegistrationFunctionName%", bundleRegistrationFunctionName) - .Replace("%AddPreallocatedResources%", addPreallocatedResources.ToString())); + .Replace("%AddPreallocatedResources%", string.Join("\n", addPreallocatedResources))); } private void BundleFileToCSource(string symbolName, FileStream inputStream, StreamWriter outputUtf8Writer) @@ -370,9 +370,6 @@ private void BundleFileToCSource(string symbolName, FileStream inputStream, Stre private static string ToSafeSymbolName(string destinationFileName, bool filenameOnly = true) { - // Since destinationFileName includes a content hash, we can safely strip off the directory name - // as the filename is always unique enough. This avoid disclosing information about the build - // file structure in the resulting symbols. var filename = destinationFileName; if (filenameOnly) filename = Path.GetFileName(destinationFileName); diff --git a/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template index 1352e44c047354..aea178a626f271 100644 --- a/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template +++ b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template @@ -66,4 +66,5 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t void %BundleRegistrationFunctionName% (void) { -%AddPreallocatedResources%} +%AddPreallocatedResources% +} From 0e3344214e306709c68ef0aa453863b9f79563ed Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 25 May 2023 17:16:49 -0400 Subject: [PATCH 089/110] Try adding resources with known assembly extensions as dll --- src/mono/mono/metadata/bundled-resources.c | 62 +++++----------------- 1 file changed, 13 insertions(+), 49 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index bf9397ab5f8a7b..9f1b1fdfdb431b 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -66,55 +66,11 @@ mono_bundled_resources_value_destroy_func (void *resource) value->free_bundled_resource_func (resource); } -#ifdef ENABLE_WEBCIL static bool -is_known_assembly_extension (const char *ext) -{ - return !strcmp (ext, ".dll") || !strcmp (ext, ".webcil") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); -} - -static gboolean -resource_id_equal (const char *id_one, const char *id_two) -{ - const char *extension_one = strrchr (id_one, '.'); - const char *extension_two = strrchr (id_two, '.'); - if (extension_one && extension_two && is_known_assembly_extension (extension_one) && is_known_assembly_extension (extension_two)) { - size_t len_one = extension_one - id_one; - size_t len_two = extension_two - id_two; - return (len_one == len_two) && !strncmp (id_one, id_two, len_one); - } - - return !strcmp (id_one, id_two); -} - -static guint -resource_id_hash (const char *id) +is_alternate_assembly_extension (const char *ext) { - const char *current = id; - const char *extension = NULL; - guint previous_hash = 0; - guint hash = 0; - - while (*current) { - hash = (hash << 5) - (hash + *current); - if (*current == '.') { - extension = current; - previous_hash = hash; - } - current++; - } - - // alias all extensions to .dll - if (extension && is_known_assembly_extension (extension)) { - hash = previous_hash; - hash = (hash << 5) - (hash + 'd'); - hash = (hash << 5) - (hash + 'l'); - hash = (hash << 5) - (hash + 'l'); - } - - return hash; + return !strcmp (ext, ".webcil") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); } -#endif // ENABLE_WEBCIL //--------------------------------------------------------------------------------------- // @@ -140,9 +96,9 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t { if (!bundled_resources) { #ifdef ENABLE_WEBCIL - bundled_resources = g_hash_table_new_full ((GHashFunc)resource_id_hash, (GEqualFunc)resource_id_equal, NULL, mono_bundled_resources_value_destroy_func); + bundled_resources = g_hash_table_new_full ((GHashFunc)resource_id_hash, (GEqualFunc)resource_id_equal, g_free, mono_bundled_resources_value_destroy_func); #else - bundled_resources = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, mono_bundled_resources_value_destroy_func); + bundled_resources = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, mono_bundled_resources_value_destroy_func); #endif } @@ -157,7 +113,15 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t if (resource_to_bundle->type == MONO_BUNDLED_SATELLITE_ASSEMBLY) satelliteAssemblyAdded = true; - g_hash_table_insert (bundled_resources, (gpointer) resource_to_bundle->id, resource_to_bundle); + char *key = strdup (resource_to_bundle->id); + g_assert (key); + const char *extension = strrchr (key, '.'); + if (is_alternate_assembly_extension (extension)) { + size_t n = extension - key; + memcpy (key + n, ".dll\0", 5); + } + + g_hash_table_insert (bundled_resources, key, resource_to_bundle); } if (assemblyAdded) From 37b1c8268ee4b4c5840aa6ad6d58494b8bf5a025 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 26 May 2023 16:25:55 -0400 Subject: [PATCH 090/110] Revert to custom hash and equal without WEBCIL guard This reverts commit 0e3344214e306709c68ef0aa453863b9f79563ed. --- src/mono/mono/metadata/bundled-resources.c | 62 +++++++++++++++++----- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 9f1b1fdfdb431b..bf9397ab5f8a7b 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -66,11 +66,55 @@ mono_bundled_resources_value_destroy_func (void *resource) value->free_bundled_resource_func (resource); } +#ifdef ENABLE_WEBCIL static bool -is_alternate_assembly_extension (const char *ext) +is_known_assembly_extension (const char *ext) +{ + return !strcmp (ext, ".dll") || !strcmp (ext, ".webcil") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); +} + +static gboolean +resource_id_equal (const char *id_one, const char *id_two) +{ + const char *extension_one = strrchr (id_one, '.'); + const char *extension_two = strrchr (id_two, '.'); + if (extension_one && extension_two && is_known_assembly_extension (extension_one) && is_known_assembly_extension (extension_two)) { + size_t len_one = extension_one - id_one; + size_t len_two = extension_two - id_two; + return (len_one == len_two) && !strncmp (id_one, id_two, len_one); + } + + return !strcmp (id_one, id_two); +} + +static guint +resource_id_hash (const char *id) { - return !strcmp (ext, ".webcil") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); + const char *current = id; + const char *extension = NULL; + guint previous_hash = 0; + guint hash = 0; + + while (*current) { + hash = (hash << 5) - (hash + *current); + if (*current == '.') { + extension = current; + previous_hash = hash; + } + current++; + } + + // alias all extensions to .dll + if (extension && is_known_assembly_extension (extension)) { + hash = previous_hash; + hash = (hash << 5) - (hash + 'd'); + hash = (hash << 5) - (hash + 'l'); + hash = (hash << 5) - (hash + 'l'); + } + + return hash; } +#endif // ENABLE_WEBCIL //--------------------------------------------------------------------------------------- // @@ -96,9 +140,9 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t { if (!bundled_resources) { #ifdef ENABLE_WEBCIL - bundled_resources = g_hash_table_new_full ((GHashFunc)resource_id_hash, (GEqualFunc)resource_id_equal, g_free, mono_bundled_resources_value_destroy_func); + bundled_resources = g_hash_table_new_full ((GHashFunc)resource_id_hash, (GEqualFunc)resource_id_equal, NULL, mono_bundled_resources_value_destroy_func); #else - bundled_resources = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, mono_bundled_resources_value_destroy_func); + bundled_resources = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, mono_bundled_resources_value_destroy_func); #endif } @@ -113,15 +157,7 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t if (resource_to_bundle->type == MONO_BUNDLED_SATELLITE_ASSEMBLY) satelliteAssemblyAdded = true; - char *key = strdup (resource_to_bundle->id); - g_assert (key); - const char *extension = strrchr (key, '.'); - if (is_alternate_assembly_extension (extension)) { - size_t n = extension - key; - memcpy (key + n, ".dll\0", 5); - } - - g_hash_table_insert (bundled_resources, key, resource_to_bundle); + g_hash_table_insert (bundled_resources, (gpointer) resource_to_bundle->id, resource_to_bundle); } if (assemblyAdded) From 88f90d431e2abd641a9590f6989fb92a709d1792 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 30 May 2023 10:33:58 -0400 Subject: [PATCH 091/110] Consolidate file and bundledResource variables --- .../EmitBundleTask/EmitBundleBase.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index 24152a88792bf2..9fb5c0e0f01a55 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -57,25 +57,23 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab public override bool Execute() { List bundledResources = new(FilesToBundle.Length); - foreach (ITaskItem file in FilesToBundle) + foreach (ITaskItem bundledResource in FilesToBundle) { - ITaskItem bundledResource = file; - - var registeredName = file.GetMetadata("RegisteredName"); + var registeredName = bundledResource.GetMetadata("RegisteredName"); if (string.IsNullOrEmpty(registeredName)) { - registeredName = Path.GetFileName(file.ItemSpec); - file.SetMetadata("RegisteredName", registeredName); + registeredName = Path.GetFileName(bundledResource.ItemSpec); + bundledResource.SetMetadata("RegisteredName", registeredName); } - string resourceDataSymbol = $"bundled_resource_{ToSafeSymbolName(Utils.ComputeHash(file.ItemSpec))}"; + string resourceDataSymbol = $"bundled_resource_{ToSafeSymbolName(Utils.ComputeHash(bundledResource.ItemSpec))}"; if (resourceDataSymbolDictionary.ContainsKey(registeredName)) { throw new LogAsErrorException($"Multiple resources have the same RegisteredName '{registeredName}'. Ensure {nameof(FilesToBundle)} 'RegisteredName' metadata are set and unique."); } resourceDataSymbolDictionary.Add(registeredName, resourceDataSymbol); - file.SetMetadata("DestinationFile", Path.Combine(OutputDirectory, resourceDataSymbol + GetDestinationFileExtension())); + bundledResource.SetMetadata("DestinationFile", Path.Combine(OutputDirectory, resourceDataSymbol + GetDestinationFileExtension())); string[] resourcesWithDataSymbol; if (resourcesForDataSymbolDictionary.TryGetValue(resourceDataSymbol, out string[]? resourcesAlreadyWithDataSymbol)) From e4c6ef6535f4f1c779355e8008ae031c3d0be69d Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 30 May 2023 10:38:55 -0400 Subject: [PATCH 092/110] Add culture metadata to SatelliteAssembly resources --- .../EmitBundleTask/EmitBundleBase.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index 9fb5c0e0f01a55..364a9824abd77f 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -62,7 +62,15 @@ public override bool Execute() var registeredName = bundledResource.GetMetadata("RegisteredName"); if (string.IsNullOrEmpty(registeredName)) { - registeredName = Path.GetFileName(bundledResource.ItemSpec); + string culture = bundledResource.GetMetadata("Culture"); + if (!string.IsNullOrEmpty(culture)) + { + registeredName = culture + "/" + Path.GetFileName(bundledResource.ItemSpec); + } + else + { + registeredName = Path.GetFileName(bundledResource.ItemSpec); + } bundledResource.SetMetadata("RegisteredName", registeredName); } @@ -253,7 +261,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r { preloadedStruct = satelliteAssemblyTemplate; preloadedStruct.Replace("%Culture%", tuple.culture); - resourceId = $"{tuple.culture}/{tuple.registeredName}"; + resourceId = tuple.registeredName; preallocatedSatelliteAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); satelliteAssembliesCount += 1; } From dbe7c21de7da9a73ab1f5356118d22a1cb846f93 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 30 May 2023 11:39:30 -0400 Subject: [PATCH 093/110] Truncate Encoding for readability --- src/tasks/Common/Utils.cs | 83 ++++++++++++++++++- .../EmitBundleTask/EmitBundleBase.cs | 14 +++- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index 2f1cbd98fb3b66..b4a63116ed81ee 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -16,6 +16,19 @@ internal static class Utils { + public enum HashAlgorithmType + { + SHA256, + SHA384, + SHA512 + }; + + public enum HashEncodingType + { + Base64, + Base64Safe + }; + public static string WebcilInWasmExtension = ".wasm"; private static readonly object s_SyncObj = new object(); @@ -213,13 +226,75 @@ public static bool CopyIfDifferent(string src, string dst, bool useHash) return areDifferent; } + private static string ToBase64SafeString(byte[] data) + { + if (data.Length == 0) + return string.Empty; + + int outputLength = ((4 * data.Length / 3) + 3) & ~3; + char[] base64Safe = new char[outputLength]; + int base64SafeLength = Convert.ToBase64CharArray(data, 0, data.Length, base64Safe, 0, Base64FormattingOptions.None); + + //RFC3548, URL and Filename Safe Alphabet. + for (int i = 0; i < base64SafeLength; i++) + { + if (base64Safe[i] == '+') + base64Safe[i] = '-'; + else if (base64Safe[i] == '/') + base64Safe[i] = '_'; + } + + return new string(base64Safe); + } + + private static byte[] ComputeHashFromStream(Stream stream, HashAlgorithmType algorithm) + { + if (algorithm == HashAlgorithmType.SHA512) + { + using HashAlgorithm hashAlgorithm = SHA512.Create(); + return hashAlgorithm.ComputeHash(stream); + } + else if (algorithm == HashAlgorithmType.SHA384) + { + using HashAlgorithm hashAlgorithm = SHA384.Create(); + return hashAlgorithm.ComputeHash(stream); + } + else if (algorithm == HashAlgorithmType.SHA256) + { + using HashAlgorithm hashAlgorithm = SHA256.Create(); + return hashAlgorithm.ComputeHash(stream); + } + else + { + throw new ArgumentException($"Unsupported hash algorithm: {algorithm}"); + } + } + + private static string EncodeHash(byte[] data, HashEncodingType encoding) + { + if (encoding == HashEncodingType.Base64) + { + return Convert.ToBase64String(data); + } + else if (encoding == HashEncodingType.Base64Safe) + { + return ToBase64SafeString(data); + } + else + { + throw new ArgumentException($"Unsupported hash encoding: {encoding}"); + } + } + public static string ComputeHash(string filepath) { - using var stream = File.OpenRead(filepath); - using HashAlgorithm hashAlgorithm = SHA512.Create(); + return ComputeHashEx(filepath); + } - byte[] hash = hashAlgorithm.ComputeHash(stream); - return Convert.ToBase64String(hash); + public static string ComputeHashEx(string filepath, HashAlgorithmType algorithm = HashAlgorithmType.SHA512, HashEncodingType encoding = HashEncodingType.Base64) + { + using var stream = File.OpenRead(filepath); + return EncodeHash(ComputeHashFromStream(stream, algorithm), encoding); } public static string ComputeIntegrity(string filepath) diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index 364a9824abd77f..1edae210945ca3 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -21,6 +21,10 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab private Dictionary resourcesForDataSymbolDictionary= new(); + /// Truncate encoded hashes used in file names and symbols to 24 characters. + /// Represents a 128-bit hash output encoded in base64 format. + private const int MaxEncodedHashLength = 24; + /// Must have DestinationFile metadata, which is the output filename /// Could have RegisteredName, otherwise it would be the filename. /// RegisteredName should be prefixed with namespace in form of unix like path. For example: "/usr/share/zoneinfo/" @@ -74,7 +78,7 @@ public override bool Execute() bundledResource.SetMetadata("RegisteredName", registeredName); } - string resourceDataSymbol = $"bundled_resource_{ToSafeSymbolName(Utils.ComputeHash(bundledResource.ItemSpec))}"; + string resourceDataSymbol = $"bundled_resource_{ToSafeSymbolName(TruncateEncodedHash(Utils.ComputeHashEx(bundledResource.ItemSpec, Utils.HashAlgorithmType.SHA256, Utils.HashEncodingType.Base64Safe), MaxEncodedHashLength))}"; if (resourceDataSymbolDictionary.ContainsKey(registeredName)) { throw new LogAsErrorException($"Multiple resources have the same RegisteredName '{registeredName}'. Ensure {nameof(FilesToBundle)} 'RegisteredName' metadata are set and unique."); @@ -391,6 +395,14 @@ private static string ToSafeSymbolName(string destinationFileName, bool filename return sb.ToString(); } + private static string TruncateEncodedHash(string encodedHash, int maxEncodedHashLength) + { + if (string.IsNullOrEmpty(encodedHash)) + return string.Empty; + + return encodedHash.Substring(0, Math.Min(encodedHash.Length, maxEncodedHashLength)); + } + // Equivalent to "isalnum" private static bool IsAlphanumeric(char c) => c is (>= 'a' and <= 'z') From 8705c976cfd42bd8a27ed5ace57e6c907971e8cc Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 30 May 2023 16:29:41 -0400 Subject: [PATCH 094/110] Enable custom hash for bundling resources on wasm --- src/mono/mono/metadata/bundled-resources.c | 14 +++++--------- src/mono/mono/metadata/webcil-loader.h | 2 ++ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index bf9397ab5f8a7b..62162053d9c86b 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -6,7 +6,6 @@ #include #include -#include #include static GHashTable *bundled_resources = NULL; @@ -66,11 +65,14 @@ mono_bundled_resources_value_destroy_func (void *resource) value->free_bundled_resource_func (resource); } -#ifdef ENABLE_WEBCIL static bool is_known_assembly_extension (const char *ext) { +#ifdef ENABLE_WEBCIL return !strcmp (ext, ".dll") || !strcmp (ext, ".webcil") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); +#else + return !strcmp (ext, ".dll") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); +#endif } static gboolean @@ -114,7 +116,6 @@ resource_id_hash (const char *id) return hash; } -#endif // ENABLE_WEBCIL //--------------------------------------------------------------------------------------- // @@ -138,13 +139,8 @@ resource_id_hash (const char *id) void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len) { - if (!bundled_resources) { -#ifdef ENABLE_WEBCIL + if (!bundled_resources) bundled_resources = g_hash_table_new_full ((GHashFunc)resource_id_hash, (GEqualFunc)resource_id_equal, NULL, mono_bundled_resources_value_destroy_func); -#else - bundled_resources = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, mono_bundled_resources_value_destroy_func); -#endif - } bool assemblyAdded = false; bool satelliteAssemblyAdded = false; diff --git a/src/mono/mono/metadata/webcil-loader.h b/src/mono/mono/metadata/webcil-loader.h index daf62171504836..a3466356c9cb80 100644 --- a/src/mono/mono/metadata/webcil-loader.h +++ b/src/mono/mono/metadata/webcil-loader.h @@ -5,6 +5,8 @@ #ifndef _MONO_METADATA_WEBCIL_LOADER_H #define _MONO_METADATA_WEBCIL_LOADER_H +#include + #define MONO_WEBCIL_IN_WASM_EXTENSION ".wasm" void From 4071c109fe6c880fdaf3227a16061569c8138f50 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 1 Jun 2023 01:53:08 -0400 Subject: [PATCH 095/110] Leverage new bundling api for wasm wasi drivers --- src/mono/mono/component/mini-wasm-debugger.c | 2 +- src/mono/mono/metadata/bundled-resources.c | 17 +++ src/mono/mono/mini/mini-wasm.h | 2 - src/mono/wasi/runtime/driver.c | 89 +++++----------- src/mono/wasm/runtime/driver.c | 106 +++++-------------- 5 files changed, 72 insertions(+), 144 deletions(-) diff --git a/src/mono/mono/component/mini-wasm-debugger.c b/src/mono/mono/component/mini-wasm-debugger.c index 9a550eee6ba2d7..80e570f142b088 100644 --- a/src/mono/mono/component/mini-wasm-debugger.c +++ b/src/mono/mono/component/mini-wasm-debugger.c @@ -199,7 +199,7 @@ assembly_loaded (MonoProfiler *prof, MonoAssembly *assembly) return; } - if (mono_wasm_assembly_already_added(assembly->aname.name)) + if (mono_bundled_resources_get_assembly_resource_values(assembly->aname.name, NULL, NULL, NULL, NULL)) return; if (mono_has_pdb_checksum ((char *) assembly_image->raw_data, assembly_image->raw_data_len)) { //if it's a release assembly we don't need to send to DebuggerProxy diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 62162053d9c86b..c617b128cb7971 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -51,6 +51,23 @@ mono_bundled_resources_free_bundled_resource_func (void *resource) g_free (resource); } +static void +mono_wasm_free_bundled_resource_func (void *resource) +{ + MonoBundledResource *bundled_resource = (MonoBundledResource *)resource; + + if (bundled_resource->type == MONO_BUNDLED_ASSEMBLY) + g_free ((void *)bundled_resource->assembly.name) + + if (bundled_resource->type == MONO_BUNDLED_SATELLITE_ASSEMBLY) { + g_free ((void *)bundled_resource->id); + g_free ((void *)bundled_resource->satellite_assembly.name); + g_free ((void *)bundled_resource->satellite_assembly.culture); + } + + g_free (resource); +} + //--------------------------------------------------------------------------------------- // // mono_bundled_resources_value_destroy_func frees the memory allocated by the hashtable's diff --git a/src/mono/mono/mini/mini-wasm.h b/src/mono/mono/mini/mini-wasm.h index 365c178903f15f..bc34bf59b96e13 100644 --- a/src/mono/mono/mini/mini-wasm.h +++ b/src/mono/mono/mini/mini-wasm.h @@ -98,8 +98,6 @@ G_EXTERN_C void mono_wasm_enable_debugging (int log_level); void mono_wasm_set_timeout (int timeout); -int mono_wasm_assembly_already_added (const char *assembly_name); - void mono_wasm_print_stack_trace (void); gboolean diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index 4491faa19a3752..ae4903c5b53558 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -113,16 +113,6 @@ typedef SgenDescriptor MonoGCDescriptor; #include "driver-gen.c" #endif -typedef struct WasmAssembly_ WasmAssembly; - -struct WasmAssembly_ { - MonoBundledAssembly assembly; - WasmAssembly *next; -}; - -static WasmAssembly *assemblies; -static int assembly_count; - int mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned int size) { @@ -135,26 +125,24 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in mono_register_symfile_for_assembly (new_name, data, size); return 1; } - WasmAssembly *entry = g_new0 (WasmAssembly, 1); - entry->assembly.name = strdup (name); - entry->assembly.data = data; - entry->assembly.size = size; - entry->next = assemblies; - assemblies = entry; - ++assembly_count; + // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly + MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (name); + if (!assembly_resource) { + assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); + assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; + assembly_resource->resource.id = name; + assembly_resource->resource.free_bundled_resource_func = mono_wasm_free_bundled_resource_func; + mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); + } else { + // Ensure the MonoBundledAssemblyData has not been initialized + g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); + } + assembly_resource->assembly.name = strdup (name); + assembly_resource->assembly.data = (const uint8_t *)data; + assembly_resource->assembly.size = (uint32_t)size; return mono_has_pdb_checksum ((char*)data, size); } -typedef struct WasmSatelliteAssembly_ WasmSatelliteAssembly; - -struct WasmSatelliteAssembly_ { - MonoBundledSatelliteAssembly *assembly; - WasmSatelliteAssembly *next; -}; - -static WasmSatelliteAssembly *satellite_assemblies; -static int satellite_assembly_count; - char* gai_strerror(int code) { char* result = malloc(256); sprintf(result, "Error code %i", code); @@ -164,11 +152,18 @@ char* gai_strerror(int code) { void mono_wasm_add_satellite_assembly (const char *name, const char *culture, const unsigned char *data, unsigned int size) { - WasmSatelliteAssembly *entry = g_new0 (WasmSatelliteAssembly, 1); - entry->assembly = mono_create_new_bundled_satellite_assembly (name, culture, data, size); - entry->next = satellite_assemblies; - satellite_assemblies = entry; - ++satellite_assembly_count; + char *id = g_strconcat (culture, "/", name, (const char*)NULL); + MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); + g_assert (!satellite_assembly_resource); + satellite_assembly_resource = g_new0 (MonoBundledSatelliteAssemblyResource, 1); + satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; + satellite_assembly_resource->resource.id = id; + satellite_assembly_resource->resource.free_bundled_resource_func = mono_wasm_free_bundled_resource_func; + satellite_assembly_resource->satellite_assembly.name = strdup (name); + satellite_assembly_resource->satellite_assembly.culture = strdup (culture); + satellite_assembly_resource->satellite_assembly.data = (const uint8_t *)data; + satellite_assembly_resource->satellite_assembly.size = (uint32_t)size; + mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); } static void *sysglobal_native_handle; @@ -324,23 +319,6 @@ get_native_to_interp (MonoMethod *method, void *extra_arg) return addr; } -void -mono_wasm_register_bundled_satellite_assemblies (void) -{ - /* In legacy satellite_assembly_count is always false */ - if (satellite_assembly_count) { - MonoBundledSatelliteAssembly **satellite_bundle_array = g_new0 (MonoBundledSatelliteAssembly *, satellite_assembly_count + 1); - WasmSatelliteAssembly *cur = satellite_assemblies; - int i = 0; - while (cur) { - satellite_bundle_array [i] = cur->assembly; - cur = cur->next; - ++i; - } - mono_register_bundled_satellite_assemblies ((const MonoBundledSatelliteAssembly **)satellite_bundle_array); - } -} - #ifndef INVARIANT_GLOBALIZATION void load_icu_data (void) { @@ -502,19 +480,6 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mono_method_builder_ilgen_init (); mono_sgen_mono_ilgen_init (); - if (assembly_count) { - MonoBundledAssembly **bundle_array = g_new0 (MonoBundledAssembly*, assembly_count + 1); - WasmAssembly *cur = assemblies; - int i = 0; - while (cur) { - bundle_array [i] = &cur->assembly; - cur = cur->next; - ++i; - } - mono_register_bundled_assemblies ((const MonoBundledAssembly **)bundle_array); - } - - mono_wasm_register_bundled_satellite_assemblies (); mono_trace_init (); mono_trace_set_log_handler (wasi_trace_logger, NULL); diff --git a/src/mono/wasm/runtime/driver.c b/src/mono/wasm/runtime/driver.c index 91c6186f6e201c..f98bf80417849b 100644 --- a/src/mono/wasm/runtime/driver.c +++ b/src/mono/wasm/runtime/driver.c @@ -180,16 +180,6 @@ mono_wasm_deregister_root (char *addr) #include "driver-gen.c" #endif -typedef struct WasmAssembly_ WasmAssembly; - -struct WasmAssembly_ { - MonoBundledAssembly assembly; - WasmAssembly *next; -}; - -static WasmAssembly *assemblies; -static int assembly_count; - EMSCRIPTEN_KEEPALIVE int mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned int size) { @@ -201,51 +191,39 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in mono_register_symfile_for_assembly (new_name, data, size); return 1; } - WasmAssembly *entry = g_new0 (WasmAssembly, 1); - entry->assembly.name = strdup (name); - entry->assembly.data = data; - entry->assembly.size = size; - entry->next = assemblies; - assemblies = entry; - ++assembly_count; - return mono_has_pdb_checksum ((char*)data, size); -} - -int -mono_wasm_assembly_already_added (const char *assembly_name) -{ - if (assembly_count == 0) - return 0; - - WasmAssembly *entry = assemblies; - while (entry != NULL) { - int entry_name_minus_extn_len = strlen(entry->assembly.name) - 4; - if (entry_name_minus_extn_len == strlen(assembly_name) && strncmp (entry->assembly.name, assembly_name, entry_name_minus_extn_len) == 0) - return 1; - entry = entry->next; + // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly + MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (name); + if (!assembly_resource) { + assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); + assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; + assembly_resource->resource.id = name; + assembly_resource->resource.free_bundled_resource_func = mono_wasm_free_bundled_resource_func; + mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); + } else { + // Ensure the MonoBundledAssemblyData has not been initialized + g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); } - - return 0; + assembly_resource->assembly.name = strdup (name); + assembly_resource->assembly.data = (const uint8_t *)data; + assembly_resource->assembly.size = (uint32_t)size; + return mono_has_pdb_checksum ((char*)data, size); } -typedef struct WasmSatelliteAssembly_ WasmSatelliteAssembly; - -struct WasmSatelliteAssembly_ { - MonoBundledSatelliteAssembly *assembly; - WasmSatelliteAssembly *next; -}; - -static WasmSatelliteAssembly *satellite_assemblies; -static int satellite_assembly_count; - EMSCRIPTEN_KEEPALIVE void mono_wasm_add_satellite_assembly (const char *name, const char *culture, const unsigned char *data, unsigned int size) { - WasmSatelliteAssembly *entry = g_new0 (WasmSatelliteAssembly, 1); - entry->assembly = mono_create_new_bundled_satellite_assembly (name, culture, data, size); - entry->next = satellite_assemblies; - satellite_assemblies = entry; - ++satellite_assembly_count; + char *id = g_strconcat (culture, "/", name, (const char*)NULL); + MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); + g_assert (!satellite_assembly_resource); + satellite_assembly_resource = g_new0 (MonoBundledSatelliteAssemblyResource, 1); + satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; + satellite_assembly_resource->resource.id = id; + satellite_assembly_resource->resource.free_bundled_resource_func = mono_wasm_free_bundled_resource_func; + satellite_assembly_resource->satellite_assembly.name = strdup (name); + satellite_assembly_resource->satellite_assembly.culture = strdup (culture); + satellite_assembly_resource->satellite_assembly.data = (const uint8_t *)data; + satellite_assembly_resource->satellite_assembly.size = (uint32_t)size; + mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); } EMSCRIPTEN_KEEPALIVE void @@ -413,23 +391,6 @@ get_native_to_interp (MonoMethod *method, void *extra_arg) return addr; } -void -mono_wasm_register_bundled_satellite_assemblies (void) -{ - /* In legacy satellite_assembly_count is always false */ - if (satellite_assembly_count) { - MonoBundledSatelliteAssembly **satellite_bundle_array = g_new0 (MonoBundledSatelliteAssembly *, satellite_assembly_count + 1); - WasmSatelliteAssembly *cur = satellite_assemblies; - int i = 0; - while (cur) { - satellite_bundle_array [i] = cur->assembly; - cur = cur->next; - ++i; - } - mono_register_bundled_satellite_assemblies ((const MonoBundledSatelliteAssembly **)satellite_bundle_array); - } -} - void mono_wasm_link_icu_shim (void); void @@ -551,19 +512,6 @@ mono_wasm_load_runtime (const char *unused, int debug_level) mono_sgen_mono_ilgen_init (); #endif - if (assembly_count) { - MonoBundledAssembly **bundle_array = g_new0 (MonoBundledAssembly*, assembly_count + 1); - WasmAssembly *cur = assemblies; - int i = 0; - while (cur) { - bundle_array [i] = &cur->assembly; - cur = cur->next; - ++i; - } - mono_register_bundled_assemblies ((const MonoBundledAssembly **)bundle_array); - } - - mono_wasm_register_bundled_satellite_assemblies (); mono_trace_init (); mono_trace_set_log_handler (wasm_trace_logger, NULL); root_domain = mono_jit_init_version ("mono", NULL); From a573583215bfe6f779cb3b83562f8dad18314e0d Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 1 Jun 2023 01:54:52 -0400 Subject: [PATCH 096/110] Add helper to dynamically add bundled assembly and satellite_assembly resources --- src/mono/mono/metadata/assembly.c | 29 ++--------- .../metadata/bundled-resources-internals.h | 9 ++++ src/mono/mono/metadata/bundled-resources.c | 51 ++++++++++++++++--- src/mono/wasi/runtime/driver.c | 44 +++++++--------- src/mono/wasm/runtime/driver.c | 44 +++++++--------- 5 files changed, 91 insertions(+), 86 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index e011d2a0899a02..6f1de58546539c 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -3141,21 +3141,7 @@ mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) { for (int i = 0; assemblies [i]; ++i) { const MonoBundledAssembly *assembly = assemblies [i]; - // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly - MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (assembly->name); - if (!assembly_resource) { - assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); - assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; - assembly_resource->resource.id = assembly->name; - assembly_resource->resource.free_bundled_resource_func = mono_bundled_resources_free_bundled_resource_func; - mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); - } else { - // Ensure the MonoBundledAssemblyData has not been initialized - g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); - } - assembly_resource->assembly.name = assembly->name; - assembly_resource->assembly.data = (const uint8_t *)assembly->data; - assembly_resource->assembly.size = (uint32_t)assembly->size; + mono_bundled_resources_add_assembly_resource (assembly->name, (const uint8_t *)assembly->data, (uint32_t)assembly->size, mono_bundled_resources_free_bundled_resource_func); } } @@ -3186,17 +3172,8 @@ mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly * for (int i = 0; satellite_assemblies [i]; ++i) { const MonoBundledSatelliteAssembly *satellite_assembly = satellite_assemblies [i]; char *id = g_strconcat (satellite_assembly->culture, "/", satellite_assembly->name, (const char*)NULL); - MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); - g_assert (!satellite_assembly_resource); - satellite_assembly_resource = g_new0 (MonoBundledSatelliteAssemblyResource, 1); - satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; - satellite_assembly_resource->resource.id = id; - satellite_assembly_resource->resource.free_bundled_resource_func = mono_bundled_resources_free_bundled_resource_func; - satellite_assembly_resource->satellite_assembly.name = satellite_assembly->name; - satellite_assembly_resource->satellite_assembly.culture = satellite_assembly->culture; - satellite_assembly_resource->satellite_assembly.data = (const uint8_t *)satellite_assembly->data; - satellite_assembly_resource->satellite_assembly.size = (uint32_t)satellite_assembly->size; - mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); + g_assert (id); + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly->name, satellite_assembly->culture, (const uint8_t *)satellite_assembly->data, (uint32_t)satellite_assembly->size, mono_bundled_resources_free_bundled_resource_func); } } diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 8692de670e28ea..03dcf2609d465b 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -67,9 +67,18 @@ mono_bundled_resources_free (void); void mono_bundled_resources_free_bundled_resource_func (void *resource); +void +mono_wasm_free_bundled_resource_func (void *resource); + void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); +void +mono_bundled_resources_add_assembly_resource (const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); + +void +mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); + MonoBundledResource * mono_bundled_resources_get (const char *id); diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index c617b128cb7971..5cfc321cfa3c3e 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -51,18 +51,21 @@ mono_bundled_resources_free_bundled_resource_func (void *resource) g_free (resource); } -static void +void mono_wasm_free_bundled_resource_func (void *resource) { MonoBundledResource *bundled_resource = (MonoBundledResource *)resource; - if (bundled_resource->type == MONO_BUNDLED_ASSEMBLY) - g_free ((void *)bundled_resource->assembly.name) + if (bundled_resource->type == MONO_BUNDLED_ASSEMBLY) { + MonoBundledAssemblyResource *assembly_resource = (MonoBundledAssemblyResource *)bundled_resource; + g_free ((void *)assembly_resource->assembly.name); + } if (bundled_resource->type == MONO_BUNDLED_SATELLITE_ASSEMBLY) { - g_free ((void *)bundled_resource->id); - g_free ((void *)bundled_resource->satellite_assembly.name); - g_free ((void *)bundled_resource->satellite_assembly.culture); + g_free ((void *)bundled_resource->id); + MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = (MonoBundledSatelliteAssemblyResource *)bundled_resource; + g_free ((void *)satellite_assembly_resource->satellite_assembly.name); + g_free ((void *)satellite_assembly_resource->satellite_assembly.culture); } g_free (resource); @@ -180,6 +183,42 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t bundle_contains_satellite_assemblies = true; } +void +mono_bundled_resources_add_assembly_resource (const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) +{ + // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly + MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (name); + if (!assembly_resource) { + assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); + assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; + assembly_resource->resource.id = name; + assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; + mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); + } else { + // Ensure the MonoBundledAssemblyData has not been initialized + g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); + } + assembly_resource->assembly.name = name; + assembly_resource->assembly.data = data; + assembly_resource->assembly.size = size; +} + +void +mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) +{ + MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); + g_assert (!satellite_assembly_resource); + satellite_assembly_resource = g_new0 (MonoBundledSatelliteAssemblyResource, 1); + satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; + satellite_assembly_resource->resource.id = id; + satellite_assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; + satellite_assembly_resource->satellite_assembly.name = name; + satellite_assembly_resource->satellite_assembly.culture = culture; + satellite_assembly_resource->satellite_assembly.data = data; + satellite_assembly_resource->satellite_assembly.size = size; + mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); +} + //--------------------------------------------------------------------------------------- // // mono_bundled_resources_get retrieves the pointer of the MonoBundledResource associated diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index ae4903c5b53558..474ea5658f78bd 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -72,6 +72,10 @@ extern void mono_register_icu_bundle (void); #endif /* INVARIANT_GLOBALIZATION */ #endif /* WASM_SINGLE_FILE */ +extern void mono_wasm_free_bundled_resource_func (void *resource); +extern void mono_bundled_resources_add_assembly_resource (const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); +extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); + extern const char* dotnet_wasi_getentrypointassemblyname(); int32_t mono_wasi_load_icu_data(const void* pData); void load_icu_data (void); @@ -125,21 +129,9 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in mono_register_symfile_for_assembly (new_name, data, size); return 1; } - // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly - MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (name); - if (!assembly_resource) { - assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); - assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; - assembly_resource->resource.id = name; - assembly_resource->resource.free_bundled_resource_func = mono_wasm_free_bundled_resource_func; - mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); - } else { - // Ensure the MonoBundledAssemblyData has not been initialized - g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); - } - assembly_resource->assembly.name = strdup (name); - assembly_resource->assembly.data = (const uint8_t *)data; - assembly_resource->assembly.size = (uint32_t)size; + const char *assembly_name = strdup (name); + assert (assembly_name); + mono_bundled_resources_add_assembly_resource (assembly_name, data, size, mono_wasm_free_bundled_resource_func); return mono_has_pdb_checksum ((char*)data, size); } @@ -152,18 +144,16 @@ char* gai_strerror(int code) { void mono_wasm_add_satellite_assembly (const char *name, const char *culture, const unsigned char *data, unsigned int size) { - char *id = g_strconcat (culture, "/", name, (const char*)NULL); - MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); - g_assert (!satellite_assembly_resource); - satellite_assembly_resource = g_new0 (MonoBundledSatelliteAssemblyResource, 1); - satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; - satellite_assembly_resource->resource.id = id; - satellite_assembly_resource->resource.free_bundled_resource_func = mono_wasm_free_bundled_resource_func; - satellite_assembly_resource->satellite_assembly.name = strdup (name); - satellite_assembly_resource->satellite_assembly.culture = strdup (culture); - satellite_assembly_resource->satellite_assembly.data = (const uint8_t *)data; - satellite_assembly_resource->satellite_assembly.size = (uint32_t)size; - mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); + int id_len = strlen (culture) + 1 + strlen (name); // +1 is for the "/" + char *id = (char *)malloc (sizeof (char) * (id_len + 1)); // +1 is for the terminating null character + assert (id); + int num_char = snprintf (id, (id_len + 1), "%s/%s", culture, name); + assert (num_char > 0 && num_char == id_len); + const char *satellite_assembly_name = strdup (name); + assert (satellite_assembly_name); + const char *satellite_assembly_culture = strdup (culture); + assert (satellite_assembly_culture); + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, mono_wasm_free_bundled_resource_func); } static void *sysglobal_native_handle; diff --git a/src/mono/wasm/runtime/driver.c b/src/mono/wasm/runtime/driver.c index f98bf80417849b..a6ab98b442eb9d 100644 --- a/src/mono/wasm/runtime/driver.c +++ b/src/mono/wasm/runtime/driver.c @@ -57,6 +57,10 @@ extern void mono_register_timezones_bundle (void); extern void mono_wasm_set_entrypoint_breakpoint (const char* assembly_name, int method_token); static void mono_wasm_init_finalizer_thread (void); +extern void mono_wasm_free_bundled_resource_func (void *resource); +extern void mono_bundled_resources_add_assembly_resource (const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); +extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); + #ifndef DISABLE_LEGACY_JS_INTEROP #define MARSHAL_TYPE_NULL 0 @@ -191,39 +195,25 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in mono_register_symfile_for_assembly (new_name, data, size); return 1; } - // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly - MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (name); - if (!assembly_resource) { - assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); - assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; - assembly_resource->resource.id = name; - assembly_resource->resource.free_bundled_resource_func = mono_wasm_free_bundled_resource_func; - mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); - } else { - // Ensure the MonoBundledAssemblyData has not been initialized - g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); - } - assembly_resource->assembly.name = strdup (name); - assembly_resource->assembly.data = (const uint8_t *)data; - assembly_resource->assembly.size = (uint32_t)size; + const char *assembly_name = strdup (name); + assert (assembly_name); + mono_bundled_resources_add_assembly_resource (assembly_name, data, size, mono_wasm_free_bundled_resource_func); return mono_has_pdb_checksum ((char*)data, size); } EMSCRIPTEN_KEEPALIVE void mono_wasm_add_satellite_assembly (const char *name, const char *culture, const unsigned char *data, unsigned int size) { - char *id = g_strconcat (culture, "/", name, (const char*)NULL); - MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); - g_assert (!satellite_assembly_resource); - satellite_assembly_resource = g_new0 (MonoBundledSatelliteAssemblyResource, 1); - satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; - satellite_assembly_resource->resource.id = id; - satellite_assembly_resource->resource.free_bundled_resource_func = mono_wasm_free_bundled_resource_func; - satellite_assembly_resource->satellite_assembly.name = strdup (name); - satellite_assembly_resource->satellite_assembly.culture = strdup (culture); - satellite_assembly_resource->satellite_assembly.data = (const uint8_t *)data; - satellite_assembly_resource->satellite_assembly.size = (uint32_t)size; - mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); + int id_len = strlen (culture) + 1 + strlen (name); // +1 is for the "/" + char *id = (char *)malloc (sizeof (char) * (id_len + 1)); // +1 is for the terminating null character + assert (id); + int num_char = snprintf (id, (id_len + 1), "%s/%s", culture, name); + assert (num_char > 0 && num_char == id_len); + const char *satellite_assembly_name = strdup (name); + assert (satellite_assembly_name); + const char *satellite_assembly_culture = strdup (culture); + assert (satellite_assembly_culture); + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, mono_wasm_free_bundled_resource_func); } EMSCRIPTEN_KEEPALIVE void From 7d1a6bd2dc3cb5e872fa8a6ccfa7c6fc77d8e755 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 1 Jun 2023 01:46:55 -0400 Subject: [PATCH 097/110] Limit when bundled resources can be added and freed --- src/mono/mono/metadata/bundled-resources.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 5cfc321cfa3c3e..9efbbcb5f96af3 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -22,6 +23,8 @@ static bool bundle_contains_satellite_assemblies = false; void mono_bundled_resources_free (void) { + g_assert (mono_runtime_is_shutting_down ()); + g_hash_table_destroy (bundled_resources); bundled_resources = NULL; @@ -159,6 +162,9 @@ resource_id_hash (const char *id) void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len) { + MonoDomain *domain = mono_get_root_domain (); + g_assert (!domain); + if (!bundled_resources) bundled_resources = g_hash_table_new_full ((GHashFunc)resource_id_hash, (GEqualFunc)resource_id_equal, NULL, mono_bundled_resources_value_destroy_func); From 58eeb95c91c823a7f39c1db5a8defe27befcfbae Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 1 Jun 2023 17:15:43 -0400 Subject: [PATCH 098/110] Restrict MonoBundledResource getters --- .../metadata/bundled-resources-internals.h | 12 -------- src/mono/mono/metadata/bundled-resources.c | 8 +++--- src/mono/mono/metadata/mono-debug.c | 28 +++++++++---------- 3 files changed, 18 insertions(+), 30 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 03dcf2609d465b..192f4ce9219bc0 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -79,18 +79,6 @@ mono_bundled_resources_add_assembly_resource (const char *name, const uint8_t *d void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); -MonoBundledResource * -mono_bundled_resources_get (const char *id); - -MonoBundledAssemblyResource * -mono_bundled_resources_get_assembly_resource (const char *id); - -MonoBundledSatelliteAssemblyResource * -mono_bundled_resources_get_satellite_assembly_resource (const char *id); - -MonoBundledDataResource * -mono_bundled_resources_get_data_resource (const char *id); - bool mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out, const uint8_t **symbol_data_out, uint32_t *symbol_size_out); diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 9efbbcb5f96af3..f516b8c951d380 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -237,7 +237,7 @@ mono_bundled_resources_add_satellite_assembly_resource (const char *id, const ch // MonoBundledResource * - Pointer to the resource in the hashmap with the key `id` // -MonoBundledResource * +static MonoBundledResource * mono_bundled_resources_get (const char *id) { if (!bundled_resources) @@ -261,7 +261,7 @@ mono_bundled_resources_get (const char *id) // in external contexts to grab assembly and symbol data. // -MonoBundledAssemblyResource * +static MonoBundledAssemblyResource * mono_bundled_resources_get_assembly_resource (const char *id) { MonoBundledAssemblyResource *assembly = @@ -287,7 +287,7 @@ mono_bundled_resources_get_assembly_resource (const char *id) // in external contexts to grab satellite assembly data. // -MonoBundledSatelliteAssemblyResource * +static MonoBundledSatelliteAssemblyResource * mono_bundled_resources_get_satellite_assembly_resource (const char *id) { MonoBundledSatelliteAssemblyResource *satellite_assembly = @@ -313,7 +313,7 @@ mono_bundled_resources_get_satellite_assembly_resource (const char *id) // in external contexts to grab data. // -MonoBundledDataResource * +static MonoBundledDataResource * mono_bundled_resources_get_data_resource (const char *id) { MonoBundledDataResource *data = diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index a57ca41b35c3ae..7eec7a7a6053e6 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1095,9 +1095,11 @@ mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte * static MonoDebugHandle * open_symfile_from_bundle (MonoImage *image) { - MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (image->module_name); - if (assembly && assembly->symbol_data.data) - return mono_debug_open_image (image, assembly->symbol_data.data, assembly->symbol_data.size); + MonoDebugHandle *handle = NULL; + const uint8_t *data = NULL; + const uint32_t size = 0; + if (mono_bundled_resources_get_assembly_resource_values (image->module_name, &data, &size)) + handle = mono_debug_open_image (image, data, size); #ifdef ENABLE_WEBCIL size_t len = strlen (image->module_name); @@ -1108,20 +1110,22 @@ open_symfile_from_bundle (MonoImage *image) char *module_name_dll_suffix = (char *)g_malloc0 (sizeof(char) * (n + 5)); memcpy (module_name_dll_suffix, image->module_name, len); memcpy (module_name_dll_suffix + n, ".dll\0", 5); - assembly = mono_bundled_resources_get_assembly_resource (module_name_dll_suffix); + if (mono_bundled_resources_get_assembly_resource_values (module_name_dll_suffix, &data, &size)) + handle = mono_debug_open_image (image, data, size); + g_free (module_name_dll_suffix); } - if (assembly && assembly->symbol_data.data) - return mono_debug_open_image (image, assembly->symbol_data.data, assembly->symbol_data.size); #endif - return NULL; + return handle; } const mono_byte * mono_get_symfile_bytes_from_bundle (const char *assembly_name, int *size) { - MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (assembly_name); + mono_byte *data = NULL; + if (mono_bundled_resources_get_assembly_resource_values (assembly_name, &data, size)) + return data; #ifdef ENABLE_WEBCIL if (!assembly) { @@ -1133,17 +1137,13 @@ mono_get_symfile_bytes_from_bundle (const char *assembly_name, int *size) char *module_name_dll_suffix = (char *)g_malloc0 (sizeof(char) * (n + 5)); memcpy (module_name_dll_suffix, assembly_name, len); memcpy (module_name_dll_suffix + n, ".dll\0", 5); - assembly = mono_bundled_resources_get_assembly_resource (module_name_dll_suffix); + mono_bundled_resources_get_assembly_resource_values (module_name_dll_suffix, &data, size) g_free (module_name_dll_suffix); } } #endif - if (!assembly || assembly->symbol_data.data || assembly->symbol_data.size != 0) - return NULL; - - *size = assembly->symbol_data.size; - return assembly->symbol_data.data; + return data; } void From 4e568b122af75829a10532d6438b0bdd32399fc4 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 1 Jun 2023 17:50:16 -0400 Subject: [PATCH 099/110] Restrict exposure of underlying MonoBundledAssemblyResource modification --- src/mono/mono/metadata/bundled-resources.c | 105 ++++++++++++++------- 1 file changed, 70 insertions(+), 35 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index f516b8c951d380..cdcddd82d2173b 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -189,41 +189,6 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t bundle_contains_satellite_assemblies = true; } -void -mono_bundled_resources_add_assembly_resource (const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) -{ - // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly - MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (name); - if (!assembly_resource) { - assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); - assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; - assembly_resource->resource.id = name; - assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; - mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); - } else { - // Ensure the MonoBundledAssemblyData has not been initialized - g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); - } - assembly_resource->assembly.name = name; - assembly_resource->assembly.data = data; - assembly_resource->assembly.size = size; -} - -void -mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) -{ - MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); - g_assert (!satellite_assembly_resource); - satellite_assembly_resource = g_new0 (MonoBundledSatelliteAssemblyResource, 1); - satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; - satellite_assembly_resource->resource.id = id; - satellite_assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; - satellite_assembly_resource->satellite_assembly.name = name; - satellite_assembly_resource->satellite_assembly.culture = culture; - satellite_assembly_resource->satellite_assembly.data = data; - satellite_assembly_resource->satellite_assembly.size = size; - mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); -} //--------------------------------------------------------------------------------------- // @@ -425,6 +390,76 @@ mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t * return true; } +void +mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) +{ + // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly + MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (name); + if (!assembly_resource) { + assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); + assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; + assembly_resource->resource.id = id; + assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; + mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); + } else { + // Ensure the MonoBundledAssemblyData has not been initialized + g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); + } + assembly_resource->assembly.name = name; + assembly_resource->assembly.data = data; + assembly_resource->assembly.size = size; +} + +void +mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) +{ + // Check if assembly dll counterpart had been added via mono_register_bundled_assemblies + MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (id); + if (!assembly_resource) { + assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); + assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; + assembly_resource->resource.id = id; + assembly_resource->resource.free_bundled_resource_func = &mono_bundled_resources_free_bundled_resource_func; + mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); + } else { + // Ensure the MonoBundledSymbolData has not been initialized + g_assert (!assembly_resource->symbol_data.data && assembly_resource->symbol_data.size == 0); + } + assembly_resource->symbol_data.data = (const uint8_t *)raw_contents; + assembly_resource->symbol_data.size = (uint32_t)size; +} + +void +mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) +{ + MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); + g_assert (!satellite_assembly_resource); + satellite_assembly_resource = g_new0 (MonoBundledSatelliteAssemblyResource, 1); + satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; + satellite_assembly_resource->resource.id = id; + satellite_assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; + satellite_assembly_resource->satellite_assembly.name = name; + satellite_assembly_resource->satellite_assembly.culture = culture; + satellite_assembly_resource->satellite_assembly.data = data; + satellite_assembly_resource->satellite_assembly.size = size; + mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); +} + +void +mono_bundled_resources_add_data_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) +{ + MonoBundledDataResource *data_resource = mono_bundled_resources_get_data_resource (id); + g_assert (!data_resource); + data_resource = g_new0 (MonoBundledDataResource, 1); + data_resource->resource.type = MONO_BUNDLED_DATA; + data_resource->resource.id = id; + data_resource->resource.free_bundled_resource_func = free_bundled_resource_func; + data_resource->data.name = name; + data_resource->data.data = data; + data_resource->data.size = size; + mono_bundled_resources_add ((MonoBundledResource **)&data_resource, 1); +} + //--------------------------------------------------------------------------------------- // // mono_bundled_resources_contains_assemblies returns whether or not assemblies From b755fc76db4ee744def5d5578777a2925b8b95bd Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 1 Jun 2023 23:49:50 -0400 Subject: [PATCH 100/110] WIP --- src/mono/mono/metadata/assembly.c | 33 ++++++-- .../metadata/bundled-resources-internals.h | 23 +++-- src/mono/mono/metadata/bundled-resources.c | 83 ++++++++++--------- src/mono/mono/metadata/mono-debug.c | 37 ++++----- src/mono/wasi/runtime/driver.c | 14 ++-- src/mono/wasm/runtime/driver.c | 14 ++-- 6 files changed, 116 insertions(+), 88 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 6f1de58546539c..8e49d434149c9f 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1457,9 +1457,10 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M MonoImage *image = NULL; char *name = g_path_get_basename (filename); - MonoBundledAssemblyResource *assembly = mono_bundled_resources_get_assembly_resource (name); - if (assembly) - image = mono_image_open_from_data_internal (alc, (char *)assembly->assembly.data, assembly->assembly.size, FALSE, status, FALSE, name, NULL); + const uint8_t *data = NULL; + uint32_t size = 0; + if (mono_bundled_resources_get_assembly_resource_values (name, &data, &size, NULL, NULL)) + image = mono_image_open_from_data_internal (alc, (char *)data, size, FALSE, status, FALSE, name, NULL); g_free (name); return image; @@ -1474,9 +1475,10 @@ open_from_satellite_bundle (MonoAssemblyLoadContext *alc, const char *filename, MonoImage *image = NULL; char *bundle_name = g_strconcat (culture, "/", filename, (const char *)NULL); - MonoBundledSatelliteAssemblyResource *satellite_assembly = mono_bundled_resources_get_satellite_assembly_resource (bundle_name); - if (satellite_assembly) - image = mono_image_open_from_data_internal (alc, (char *)satellite_assembly->satellite_assembly.data, satellite_assembly->satellite_assembly.size, FALSE, status, FALSE, bundle_name, NULL); + const uint8_t *data = NULL; + uint32_t size = 0; + if (mono_bundled_resources_get_satellite_assembly_resource_values (bundle_name, &data, &size)) + image = mono_image_open_from_data_internal (alc, (char *)data, size, FALSE, status, FALSE, bundle_name, NULL); g_free (bundle_name); return image; @@ -3131,6 +3133,12 @@ mono_assembly_get_name_internal (MonoAssembly *assembly) return &assembly->aname; } +static void +mono_free_bundled_assembly_func (void *resource) +{ + g_free (resource); +} + /** * mono_register_bundled_assemblies: * Dynamically allocates MonoBundledAssemblyResources to leverage @@ -3141,7 +3149,7 @@ mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) { for (int i = 0; assemblies [i]; ++i) { const MonoBundledAssembly *assembly = assemblies [i]; - mono_bundled_resources_add_assembly_resource (assembly->name, (const uint8_t *)assembly->data, (uint32_t)assembly->size, mono_bundled_resources_free_bundled_resource_func); + mono_bundled_resources_add_assembly_resource (assembly->name, assembly->name, (const uint8_t *)assembly->data, (uint32_t)assembly->size, mono_free_bundled_assembly_func, NULL); } } @@ -3161,6 +3169,15 @@ mono_create_new_bundled_satellite_assembly (const char *name, const char *cultur return satellite_assembly; } +static void +mono_free_bundled_satellite_assembly_func (void *resource) +{ + MonoBundledResource *bundled_resource = (MonoBundledResource *)resource; + + g_free ((void *)bundled_resource->id); + g_free (resource); +} + /** * mono_register_bundled_satellite_assemblies: * Dynamically allocates MonoBundledSatelliteAssemblyResources to leverage @@ -3173,7 +3190,7 @@ mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly * const MonoBundledSatelliteAssembly *satellite_assembly = satellite_assemblies [i]; char *id = g_strconcat (satellite_assembly->culture, "/", satellite_assembly->name, (const char*)NULL); g_assert (id); - mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly->name, satellite_assembly->culture, (const uint8_t *)satellite_assembly->data, (uint32_t)satellite_assembly->size, mono_bundled_resources_free_bundled_resource_func); + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly->name, satellite_assembly->culture, (const uint8_t *)satellite_assembly->data, (uint32_t)satellite_assembly->size, mono_free_bundled_satellite_assembly_func, NULL); } } diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 192f4ce9219bc0..0015fdca540f60 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -19,6 +19,7 @@ typedef struct _MonoBundledResource { MonoBundledResourceType type; const char *id; void (*free_bundled_resource_func)(void *); + void *free_data; } MonoBundledResource; typedef struct _MonoBundledData { @@ -65,20 +66,14 @@ void mono_bundled_resources_free (void); void -mono_bundled_resources_free_bundled_resource_func (void *resource); +mono_wasm_free_bundled_assembly_resource_func (void *resource); void -mono_wasm_free_bundled_resource_func (void *resource); +mono_wasm_free_bundled_satellite_assembly_resource_func (void *resource); void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); -void -mono_bundled_resources_add_assembly_resource (const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); - -void -mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); - bool mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out, const uint8_t **symbol_data_out, uint32_t *symbol_size_out); @@ -88,6 +83,18 @@ mono_bundled_resources_get_satellite_assembly_resource_values (const char *id, c bool mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); +void +mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); + +void +mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); + +void +mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); + +void +mono_bundled_resources_add_data_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); + bool mono_bundled_resources_contains_assemblies (void); diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index cdcddd82d2173b..30a46f5d4d3aa7 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -32,44 +32,22 @@ mono_bundled_resources_free (void) bundle_contains_satellite_assemblies = false; } -//--------------------------------------------------------------------------------------- -// -// mono_bundled_resources_free_bundled_resource_func frees all memory allocated for a -// particular bundled resource. A MonoBundled*Resource should set the underlying -// MonoBundledResource's free_bundled_resource_func to this if it is being dynamically -// allocated (i.e. through old bundling api's). -// -// Through the old bundling api's, allocations occur for the MonoBundled*Resource. -// In addition, MonoBundledSatelliteAssemblyResource's allocate for the id. -// - void -mono_bundled_resources_free_bundled_resource_func (void *resource) +mono_wasm_free_bundled_assembly_resource_func (void *resource) { MonoBundledResource *bundled_resource = (MonoBundledResource *)resource; - - if (bundled_resource->type == MONO_BUNDLED_SATELLITE_ASSEMBLY) - g_free ((void *)bundled_resource->id); - + g_free ((void *)bundled_resource->id); g_free (resource); } void -mono_wasm_free_bundled_resource_func (void *resource) +mono_wasm_free_bundled_satellite_assembly_resource_func (void *resource) { MonoBundledResource *bundled_resource = (MonoBundledResource *)resource; - - if (bundled_resource->type == MONO_BUNDLED_ASSEMBLY) { - MonoBundledAssemblyResource *assembly_resource = (MonoBundledAssemblyResource *)bundled_resource; - g_free ((void *)assembly_resource->assembly.name); - } - - if (bundled_resource->type == MONO_BUNDLED_SATELLITE_ASSEMBLY) { - g_free ((void *)bundled_resource->id); - MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = (MonoBundledSatelliteAssemblyResource *)bundled_resource; - g_free ((void *)satellite_assembly_resource->satellite_assembly.name); - g_free ((void *)satellite_assembly_resource->satellite_assembly.culture); - } + g_free ((void *)bundled_resource->id); + MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = (MonoBundledSatelliteAssemblyResource *)resource; + g_free ((void *)satellite_assembly_resource->satellite_assembly.name); + g_free ((void *)satellite_assembly_resource->satellite_assembly.culture); g_free (resource); } @@ -84,6 +62,12 @@ static void mono_bundled_resources_value_destroy_func (void *resource) { MonoBundledResource *value = (MonoBundledResource *)resource; + if (value->free_data) { + MonoBundledResource *free_data = (MonoBundledResource *)value->free_data; + if (free_data->free_bundled_resource_func) { + free_data->free_bundled_resource_func (free_data); + } + } if (value->free_bundled_resource_func) value->free_bundled_resource_func (resource); } @@ -94,7 +78,7 @@ is_known_assembly_extension (const char *ext) #ifdef ENABLE_WEBCIL return !strcmp (ext, ".dll") || !strcmp (ext, ".webcil") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); #else - return !strcmp (ext, ".dll") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); + return !strcmp (ext, ".dll") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); #endif } @@ -391,7 +375,7 @@ mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t * } void -mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) +mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data) { // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (name); @@ -400,10 +384,21 @@ mono_bundled_resources_add_assembly_resource (const char *id, const char *name, assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; assembly_resource->resource.id = id; assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; + assembly_resource->resource.free_data = free_data; mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); } else { // Ensure the MonoBundledAssemblyData has not been initialized g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); + + MonoBundledAssemblyResource *assembly_resource_symbol_owned_data = g_new0 (MonoBundledAssemblyResource, 1); + assembly_resource_symbol_owned_data->resource.id = assembly_resource->resource.id; + assembly_resource_symbol_owned_data->resource.free_bundled_resource_func = assembly_resource->resource.free_bundled_resource_func; + assembly_resource_symbol_owned_data->resource.free_data = assembly_resource->resource.free_data; + assembly_resource_symbol_owned_data->symbol_data.data = assembly_resource->symbol_data.data; + assembly_resource_symbol_owned_data->symbol_data.size = assembly_resource->symbol_data.size; + + assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; + assembly_resource->resource.free_data = assembly_resource_symbol_owned_data; } assembly_resource->assembly.name = name; assembly_resource->assembly.data = data; @@ -411,26 +406,38 @@ mono_bundled_resources_add_assembly_resource (const char *id, const char *name, } void -mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) +mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data) { - // Check if assembly dll counterpart had been added via mono_register_bundled_assemblies + // Check if assembly dll counterpart had been added via mono_register_bundled_assemblies MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (id); if (!assembly_resource) { assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; assembly_resource->resource.id = id; - assembly_resource->resource.free_bundled_resource_func = &mono_bundled_resources_free_bundled_resource_func; + assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; + assembly_resource->resource.free_data = free_data; mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); } else { // Ensure the MonoBundledSymbolData has not been initialized g_assert (!assembly_resource->symbol_data.data && assembly_resource->symbol_data.size == 0); + + MonoBundledAssemblyResource *assembly_resource_assembly_owned_data = g_new0 (MonoBundledAssemblyResource, 1); + assembly_resource_assembly_owned_data->resource.id = assembly_resource->resource.id; + assembly_resource_assembly_owned_data->resource.free_bundled_resource_func = assembly_resource->resource.free_bundled_resource_func; + assembly_resource_assembly_owned_data->resource.free_data = assembly_resource->resource.free_data; + assembly_resource_assembly_owned_data->assembly.name = assembly_resource->assembly.name; + assembly_resource_assembly_owned_data->assembly.data = assembly_resource->assembly.data; + assembly_resource_assembly_owned_data->assembly.size = assembly_resource->assembly.size; + + assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; + assembly_resource->resource.free_data = assembly_resource_assembly_owned_data; } - assembly_resource->symbol_data.data = (const uint8_t *)raw_contents; + assembly_resource->symbol_data.data = (const uint8_t *)data; assembly_resource->symbol_data.size = (uint32_t)size; } void -mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) +mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data) { MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); g_assert (!satellite_assembly_resource); @@ -438,6 +445,7 @@ mono_bundled_resources_add_satellite_assembly_resource (const char *id, const ch satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; satellite_assembly_resource->resource.id = id; satellite_assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; + satellite_assembly_resource->resource.free_data = free_data; satellite_assembly_resource->satellite_assembly.name = name; satellite_assembly_resource->satellite_assembly.culture = culture; satellite_assembly_resource->satellite_assembly.data = data; @@ -446,7 +454,7 @@ mono_bundled_resources_add_satellite_assembly_resource (const char *id, const ch } void -mono_bundled_resources_add_data_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)) +mono_bundled_resources_add_data_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data) { MonoBundledDataResource *data_resource = mono_bundled_resources_get_data_resource (id); g_assert (!data_resource); @@ -454,6 +462,7 @@ mono_bundled_resources_add_data_resource (const char *id, const char *name, cons data_resource->resource.type = MONO_BUNDLED_DATA; data_resource->resource.id = id; data_resource->resource.free_bundled_resource_func = free_bundled_resource_func; + data_resource->resource.free_data = free_data; data_resource->data.name = name; data_resource->data.data = data; data_resource->data.size = size; diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 7eec7a7a6053e6..12cdc017422daa 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1076,20 +1076,7 @@ mono_is_debugger_attached (void) void mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte *raw_contents, int size) { - // Check if assembly dll counterpart had been added via mono_register_bundled_assemblies - MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (assembly_name); - if (!assembly_resource) { - assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); - assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; - assembly_resource->resource.id = assembly_name; - assembly_resource->resource.free_bundled_resource_func = &mono_bundled_resources_free_bundled_resource_func; - mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); - } else { - // Ensure the MonoBundledSymbolData has not been initialized - g_assert (!assembly_resource->symbol_data.data && assembly_resource->symbol_data.size == 0); - } - assembly_resource->symbol_data.data = (const uint8_t *)raw_contents; - assembly_resource->symbol_data.size = (uint32_t)size; + mono_bundled_resources_add_assembly_symbol_resource (assembly_name, raw_contents, size, NULL, NULL); } static MonoDebugHandle * @@ -1097,8 +1084,8 @@ open_symfile_from_bundle (MonoImage *image) { MonoDebugHandle *handle = NULL; const uint8_t *data = NULL; - const uint32_t size = 0; - if (mono_bundled_resources_get_assembly_resource_values (image->module_name, &data, &size)) + uint32_t size = 0; + if (mono_bundled_resources_get_assembly_resource_values (image->module_name, NULL, NULL, &data, &size)) handle = mono_debug_open_image (image, data, size); #ifdef ENABLE_WEBCIL @@ -1110,7 +1097,7 @@ open_symfile_from_bundle (MonoImage *image) char *module_name_dll_suffix = (char *)g_malloc0 (sizeof(char) * (n + 5)); memcpy (module_name_dll_suffix, image->module_name, len); memcpy (module_name_dll_suffix + n, ".dll\0", 5); - if (mono_bundled_resources_get_assembly_resource_values (module_name_dll_suffix, &data, &size)) + if (mono_bundled_resources_get_assembly_resource_values (module_name_dll_suffix, NULL, NULL, &data, &size)) handle = mono_debug_open_image (image, data, size); g_free (module_name_dll_suffix); @@ -1123,12 +1110,15 @@ open_symfile_from_bundle (MonoImage *image) const mono_byte * mono_get_symfile_bytes_from_bundle (const char *assembly_name, int *size) { - mono_byte *data = NULL; - if (mono_bundled_resources_get_assembly_resource_values (assembly_name, &data, size)) - return data; + const uint8_t *symbol_data = NULL; + uint32_t symbol_size = 0; + if (mono_bundled_resources_get_assembly_resource_values (assembly_name, NULL, NULL, &symbol_data, &symbol_size)) { + *size = symbol_size; + return (mono_byte *)symbol_data; + } #ifdef ENABLE_WEBCIL - if (!assembly) { + if (!symbol_data) { size_t len = strlen (assembly_name); char *extension = strrchr (assembly_name, '.'); /* if image's module_name ends with an acceptable extension, check if theres a bundled resource with a .dll extension instead */ @@ -1137,13 +1127,14 @@ mono_get_symfile_bytes_from_bundle (const char *assembly_name, int *size) char *module_name_dll_suffix = (char *)g_malloc0 (sizeof(char) * (n + 5)); memcpy (module_name_dll_suffix, assembly_name, len); memcpy (module_name_dll_suffix + n, ".dll\0", 5); - mono_bundled_resources_get_assembly_resource_values (module_name_dll_suffix, &data, size) + mono_bundled_resources_get_assembly_resource_values (module_name_dll_suffix, NULL, NULL, &symbol_data, &symbol_size); g_free (module_name_dll_suffix); } } #endif - return data; + *size = symbol_size; + return (mono_byte *)symbol_data; } void diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index 474ea5658f78bd..d6a7f6abdee5ca 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -72,9 +72,11 @@ extern void mono_register_icu_bundle (void); #endif /* INVARIANT_GLOBALIZATION */ #endif /* WASM_SINGLE_FILE */ -extern void mono_wasm_free_bundled_resource_func (void *resource); -extern void mono_bundled_resources_add_assembly_resource (const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); -extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); +extern void mono_wasm_free_bundled_assembly_resource_func (void *resource); +extern void mono_wasm_free_bundled_satellite_assembly_resource_func (void *resource); +extern void mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); +extern void mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); +extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); extern const char* dotnet_wasi_getentrypointassemblyname(); int32_t mono_wasi_load_icu_data(const void* pData); @@ -126,12 +128,12 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in char *new_name = strdup (name); //FIXME handle debugging assemblies with .exe extension strcpy (&new_name [len - 3], "dll"); - mono_register_symfile_for_assembly (new_name, data, size); + mono_bundled_resources_add_assembly_symbol_resource (new_name, data, size, mono_wasm_free_bundled_assembly_resource_func, NULL); return 1; } const char *assembly_name = strdup (name); assert (assembly_name); - mono_bundled_resources_add_assembly_resource (assembly_name, data, size, mono_wasm_free_bundled_resource_func); + mono_bundled_resources_add_assembly_resource (assembly_name, assembly_name, data, size, mono_wasm_free_bundled_assembly_resource_func, NULL); return mono_has_pdb_checksum ((char*)data, size); } @@ -153,7 +155,7 @@ mono_wasm_add_satellite_assembly (const char *name, const char *culture, const u assert (satellite_assembly_name); const char *satellite_assembly_culture = strdup (culture); assert (satellite_assembly_culture); - mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, mono_wasm_free_bundled_resource_func); + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, mono_wasm_free_bundled_satellite_assembly_resource_func, NULL); } static void *sysglobal_native_handle; diff --git a/src/mono/wasm/runtime/driver.c b/src/mono/wasm/runtime/driver.c index a6ab98b442eb9d..a90d7de8d9a4a0 100644 --- a/src/mono/wasm/runtime/driver.c +++ b/src/mono/wasm/runtime/driver.c @@ -57,9 +57,11 @@ extern void mono_register_timezones_bundle (void); extern void mono_wasm_set_entrypoint_breakpoint (const char* assembly_name, int method_token); static void mono_wasm_init_finalizer_thread (void); -extern void mono_wasm_free_bundled_resource_func (void *resource); -extern void mono_bundled_resources_add_assembly_resource (const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); -extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *)); +extern void mono_wasm_free_bundled_assembly_resource_func (void *resource); +extern void mono_wasm_free_bundled_satellite_assembly_resource_func (void *resource); +extern void mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); +extern void mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); +extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); #ifndef DISABLE_LEGACY_JS_INTEROP @@ -192,12 +194,12 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in char *new_name = strdup (name); //FIXME handle debugging assemblies with .exe extension strcpy (&new_name [len - 3], "dll"); - mono_register_symfile_for_assembly (new_name, data, size); + mono_bundled_resources_add_assembly_symbol_resource (new_name, data, size, mono_wasm_free_bundled_assembly_resource_func, NULL); return 1; } const char *assembly_name = strdup (name); assert (assembly_name); - mono_bundled_resources_add_assembly_resource (assembly_name, data, size, mono_wasm_free_bundled_resource_func); + mono_bundled_resources_add_assembly_resource (assembly_name, assembly_name, data, size, mono_wasm_free_bundled_assembly_resource_func, NULL); return mono_has_pdb_checksum ((char*)data, size); } @@ -213,7 +215,7 @@ mono_wasm_add_satellite_assembly (const char *name, const char *culture, const u assert (satellite_assembly_name); const char *satellite_assembly_culture = strdup (culture); assert (satellite_assembly_culture); - mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, mono_wasm_free_bundled_resource_func); + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, mono_wasm_free_bundled_satellite_assembly_resource_func, NULL); } EMSCRIPTEN_KEEPALIVE void From e34aff7a807e475ecba252ec4e45e0407bc248bf Mon Sep 17 00:00:00 2001 From: lateralusX Date: Fri, 2 Jun 2023 13:31:46 +0200 Subject: [PATCH 101/110] Free function chaining. --- src/mono/mono/metadata/assembly.c | 17 +- .../metadata/bundled-resources-internals.h | 20 +- src/mono/mono/metadata/bundled-resources.c | 197 ++++++++++-------- src/mono/wasi/runtime/driver.c | 41 +++- src/mono/wasm/runtime/driver.c | 35 +++- 5 files changed, 181 insertions(+), 129 deletions(-) diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 8e49d434149c9f..c5e5f6f8f7093a 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -3133,12 +3133,6 @@ mono_assembly_get_name_internal (MonoAssembly *assembly) return &assembly->aname; } -static void -mono_free_bundled_assembly_func (void *resource) -{ - g_free (resource); -} - /** * mono_register_bundled_assemblies: * Dynamically allocates MonoBundledAssemblyResources to leverage @@ -3149,7 +3143,7 @@ mono_register_bundled_assemblies (const MonoBundledAssembly **assemblies) { for (int i = 0; assemblies [i]; ++i) { const MonoBundledAssembly *assembly = assemblies [i]; - mono_bundled_resources_add_assembly_resource (assembly->name, assembly->name, (const uint8_t *)assembly->data, (uint32_t)assembly->size, mono_free_bundled_assembly_func, NULL); + mono_bundled_resources_add_assembly_resource (assembly->name, assembly->name, (const uint8_t *)assembly->data, (uint32_t)assembly->size, NULL, NULL); } } @@ -3170,12 +3164,9 @@ mono_create_new_bundled_satellite_assembly (const char *name, const char *cultur } static void -mono_free_bundled_satellite_assembly_func (void *resource) +mono_free_bundled_satellite_assembly_func (void *resource, void *free_data) { - MonoBundledResource *bundled_resource = (MonoBundledResource *)resource; - - g_free ((void *)bundled_resource->id); - g_free (resource); + g_free (free_data); } /** @@ -3190,7 +3181,7 @@ mono_register_bundled_satellite_assemblies (const MonoBundledSatelliteAssembly * const MonoBundledSatelliteAssembly *satellite_assembly = satellite_assemblies [i]; char *id = g_strconcat (satellite_assembly->culture, "/", satellite_assembly->name, (const char*)NULL); g_assert (id); - mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly->name, satellite_assembly->culture, (const uint8_t *)satellite_assembly->data, (uint32_t)satellite_assembly->size, mono_free_bundled_satellite_assembly_func, NULL); + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly->name, satellite_assembly->culture, (const uint8_t *)satellite_assembly->data, (uint32_t)satellite_assembly->size, mono_free_bundled_satellite_assembly_func, id); } } diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 0015fdca540f60..5ece7ea507f6b5 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -15,10 +15,12 @@ typedef enum { MONO_BUNDLED_RESOURCE_COUNT } MonoBundledResourceType; +typedef void (*free_bundled_resource_func)(void *, void*); + typedef struct _MonoBundledResource { MonoBundledResourceType type; const char *id; - void (*free_bundled_resource_func)(void *); + free_bundled_resource_func free_func; void *free_data; } MonoBundledResource; @@ -65,12 +67,6 @@ typedef struct _MonoBundledSatelliteAssemblyResource { void mono_bundled_resources_free (void); -void -mono_wasm_free_bundled_assembly_resource_func (void *resource); - -void -mono_wasm_free_bundled_satellite_assembly_resource_func (void *resource); - void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); @@ -84,16 +80,16 @@ bool mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); void -mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); +mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data); void -mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); +mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data); void -mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); +mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data); void -mono_bundled_resources_add_data_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); +mono_bundled_resources_add_data_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data); bool mono_bundled_resources_contains_assemblies (void); @@ -101,4 +97,4 @@ mono_bundled_resources_contains_assemblies (void); bool mono_bundled_resources_contains_satellite_assemblies (void); -#endif /* __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ */ \ No newline at end of file +#endif /* __MONO_METADATA_BUNDLED_RESOURCES_INTERNALS_H__ */ diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 30a46f5d4d3aa7..fcc832986a3fb9 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -10,8 +10,14 @@ #include static GHashTable *bundled_resources = NULL; -static bool bundle_contains_assemblies = false; -static bool bundle_contains_satellite_assemblies = false; +static bool bundled_resources_contains_assemblies = false; +static bool bundled_resources_contains_satellite_assemblies = false; + +typedef struct _BundledResourcesChainedFreeFunc { + free_bundled_resource_func free_func; + void *free_data; + void *next; +} BundledResourcesChainedFreeFunc; //--------------------------------------------------------------------------------------- // @@ -28,52 +34,26 @@ mono_bundled_resources_free (void) g_hash_table_destroy (bundled_resources); bundled_resources = NULL; - bundle_contains_assemblies = false; - bundle_contains_satellite_assemblies = false; -} - -void -mono_wasm_free_bundled_assembly_resource_func (void *resource) -{ - MonoBundledResource *bundled_resource = (MonoBundledResource *)resource; - g_free ((void *)bundled_resource->id); - g_free (resource); -} - -void -mono_wasm_free_bundled_satellite_assembly_resource_func (void *resource) -{ - MonoBundledResource *bundled_resource = (MonoBundledResource *)resource; - g_free ((void *)bundled_resource->id); - MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = (MonoBundledSatelliteAssemblyResource *)resource; - g_free ((void *)satellite_assembly_resource->satellite_assembly.name); - g_free ((void *)satellite_assembly_resource->satellite_assembly.culture); - - g_free (resource); + bundled_resources_contains_assemblies = false; + bundled_resources_contains_satellite_assemblies = false; } //--------------------------------------------------------------------------------------- // -// mono_bundled_resources_value_destroy_func frees the memory allocated by the hashtable's +// bundled_resources_value_destroy_func frees the memory allocated by the hashtable's // MonoBundled*Resource by invoking its underlying free_bundled_resource_func when possible. // static void -mono_bundled_resources_value_destroy_func (void *resource) +bundled_resources_value_destroy_func (void *resource) { MonoBundledResource *value = (MonoBundledResource *)resource; - if (value->free_data) { - MonoBundledResource *free_data = (MonoBundledResource *)value->free_data; - if (free_data->free_bundled_resource_func) { - free_data->free_bundled_resource_func (free_data); - } - } - if (value->free_bundled_resource_func) - value->free_bundled_resource_func (resource); + if (value->free_func) + value->free_func (resource, value->free_data); } static bool -is_known_assembly_extension (const char *ext) +bundled_resources_is_known_assembly_extension (const char *ext) { #ifdef ENABLE_WEBCIL return !strcmp (ext, ".dll") || !strcmp (ext, ".webcil") || !strcmp (ext, MONO_WEBCIL_IN_WASM_EXTENSION); @@ -83,11 +63,11 @@ is_known_assembly_extension (const char *ext) } static gboolean -resource_id_equal (const char *id_one, const char *id_two) +bundled_resources_resource_id_equal (const char *id_one, const char *id_two) { const char *extension_one = strrchr (id_one, '.'); const char *extension_two = strrchr (id_two, '.'); - if (extension_one && extension_two && is_known_assembly_extension (extension_one) && is_known_assembly_extension (extension_two)) { + if (extension_one && extension_two && bundled_resources_is_known_assembly_extension (extension_one) && bundled_resources_is_known_assembly_extension (extension_two)) { size_t len_one = extension_one - id_one; size_t len_two = extension_two - id_two; return (len_one == len_two) && !strncmp (id_one, id_two, len_one); @@ -97,7 +77,7 @@ resource_id_equal (const char *id_one, const char *id_two) } static guint -resource_id_hash (const char *id) +bundled_resources_resource_id_hash (const char *id) { const char *current = id; const char *extension = NULL; @@ -114,7 +94,7 @@ resource_id_hash (const char *id) } // alias all extensions to .dll - if (extension && is_known_assembly_extension (extension)) { + if (extension && bundled_resources_is_known_assembly_extension (extension)) { hash = previous_hash; hash = (hash << 5) - (hash + 'd'); hash = (hash << 5) - (hash + 'l'); @@ -150,7 +130,7 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t g_assert (!domain); if (!bundled_resources) - bundled_resources = g_hash_table_new_full ((GHashFunc)resource_id_hash, (GEqualFunc)resource_id_equal, NULL, mono_bundled_resources_value_destroy_func); + bundled_resources = g_hash_table_new_full ((GHashFunc)bundled_resources_resource_id_hash, (GEqualFunc)bundled_resources_resource_id_equal, NULL, bundled_resources_value_destroy_func); bool assemblyAdded = false; bool satelliteAssemblyAdded = false; @@ -167,10 +147,10 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t } if (assemblyAdded) - bundle_contains_assemblies = true; + bundled_resources_contains_assemblies = true; if (satelliteAssemblyAdded) - bundle_contains_satellite_assemblies = true; + bundled_resources_contains_satellite_assemblies = true; } @@ -187,7 +167,7 @@ mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t // static MonoBundledResource * -mono_bundled_resources_get (const char *id) +bundled_resources_get (const char *id) { if (!bundled_resources) return NULL; @@ -211,10 +191,10 @@ mono_bundled_resources_get (const char *id) // static MonoBundledAssemblyResource * -mono_bundled_resources_get_assembly_resource (const char *id) +bundled_resources_get_assembly_resource (const char *id) { MonoBundledAssemblyResource *assembly = - (MonoBundledAssemblyResource*)mono_bundled_resources_get (id); + (MonoBundledAssemblyResource*)bundled_resources_get (id); if (!assembly) return NULL; g_assert (assembly->resource.type == MONO_BUNDLED_ASSEMBLY); @@ -237,10 +217,10 @@ mono_bundled_resources_get_assembly_resource (const char *id) // static MonoBundledSatelliteAssemblyResource * -mono_bundled_resources_get_satellite_assembly_resource (const char *id) +bundled_resources_get_satellite_assembly_resource (const char *id) { MonoBundledSatelliteAssemblyResource *satellite_assembly = - (MonoBundledSatelliteAssemblyResource*)mono_bundled_resources_get (id); + (MonoBundledSatelliteAssemblyResource*)bundled_resources_get (id); if (!satellite_assembly) return NULL; g_assert (satellite_assembly->resource.type == MONO_BUNDLED_SATELLITE_ASSEMBLY); @@ -263,10 +243,10 @@ mono_bundled_resources_get_satellite_assembly_resource (const char *id) // static MonoBundledDataResource * -mono_bundled_resources_get_data_resource (const char *id) +bundled_resources_get_data_resource (const char *id) { MonoBundledDataResource *data = - (MonoBundledDataResource*)mono_bundled_resources_get (id); + (MonoBundledDataResource*)bundled_resources_get (id); if (!data) return NULL; g_assert (data->resource.type == MONO_BUNDLED_DATA); @@ -296,7 +276,7 @@ mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8 *size_out = 0; *symbol_data_out = NULL; *size_out = 0; - MonoBundledAssemblyResource *bundled_assembly_resource = mono_bundled_resources_get_assembly_resource (id); + MonoBundledAssemblyResource *bundled_assembly_resource = bundled_resources_get_assembly_resource (id); if (!bundled_assembly_resource || !bundled_assembly_resource->assembly.data || bundled_assembly_resource->assembly.size == 0) @@ -330,7 +310,7 @@ mono_bundled_resources_get_satellite_assembly_resource_values (const char *id, c *data_out = NULL; *size_out = 0; - MonoBundledSatelliteAssemblyResource *bundled_satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); + MonoBundledSatelliteAssemblyResource *bundled_satellite_assembly_resource = bundled_resources_get_satellite_assembly_resource (id); if (!bundled_satellite_assembly_resource || !bundled_satellite_assembly_resource->satellite_assembly.data || bundled_satellite_assembly_resource->satellite_assembly.size == 0) @@ -362,7 +342,7 @@ mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t * *data_out = NULL; *size_out = 0; - MonoBundledDataResource *bundled_data_resource = mono_bundled_resources_get_data_resource (id); + MonoBundledDataResource *bundled_data_resource = bundled_resources_get_data_resource (id); if (!bundled_data_resource || !bundled_data_resource->data.data || bundled_data_resource->data.size == 0) @@ -374,31 +354,72 @@ mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t * return true; } +static void +bundled_resources_chained_free_func (void *resource, void *free_data) +{ + BundledResourcesChainedFreeFunc *node = (BundledResourcesChainedFreeFunc *)free_data; + if (node && node->free_func) + node->free_func (resource, node->free_data); + if (node && node->next) + bundled_resources_chained_free_func (resource, node->next); + + g_free (free_data); +} + +static void +bundled_resources_free_func (void *resource, void *free_data) +{ + bundled_resources_chained_free_func (resource, free_data); + g_free (resource); +} + +static void +bundled_resource_add_free_func (MonoBundledResource *resource, free_bundled_resource_func free_func, void *free_data) +{ + if (!free_func) + return; + + if (!resource->free_func) { + resource->free_func = free_func; + resource->free_data = free_data; + } else if (resource->free_func == bundled_resources_chained_free_func || resource->free_func == bundled_resources_free_func) { + BundledResourcesChainedFreeFunc *node = g_new0 (BundledResourcesChainedFreeFunc, 1); + node->free_func = free_func; + node->free_data = free_data; + node->next = resource->free_data; + resource->free_data = node; + } else { + BundledResourcesChainedFreeFunc *node1 = g_new0 (BundledResourcesChainedFreeFunc, 1); + BundledResourcesChainedFreeFunc *node2 = g_new0 (BundledResourcesChainedFreeFunc, 2); + + node2->free_func = resource->free_func; + node2->free_data = resource->free_data; + + node1->free_func = free_func; + node1->free_data = free_data; + node1->next = node2; + + resource->free_func = bundled_resources_chained_free_func; + resource->free_data = node1; + } +} + void -mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data) +mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data) { // Check if assembly pdb counterpart had been added via mono_register_symfile_for_assembly - MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (name); + MonoBundledAssemblyResource *assembly_resource = bundled_resources_get_assembly_resource (name); if (!assembly_resource) { assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; assembly_resource->resource.id = id; - assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; - assembly_resource->resource.free_data = free_data; + assembly_resource->resource.free_func = bundled_resources_free_func; + bundled_resource_add_free_func ((MonoBundledResource *)assembly_resource, free_func, free_data); mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); } else { // Ensure the MonoBundledAssemblyData has not been initialized g_assert (!assembly_resource->assembly.name && !assembly_resource->assembly.data && assembly_resource->assembly.size == 0); - - MonoBundledAssemblyResource *assembly_resource_symbol_owned_data = g_new0 (MonoBundledAssemblyResource, 1); - assembly_resource_symbol_owned_data->resource.id = assembly_resource->resource.id; - assembly_resource_symbol_owned_data->resource.free_bundled_resource_func = assembly_resource->resource.free_bundled_resource_func; - assembly_resource_symbol_owned_data->resource.free_data = assembly_resource->resource.free_data; - assembly_resource_symbol_owned_data->symbol_data.data = assembly_resource->symbol_data.data; - assembly_resource_symbol_owned_data->symbol_data.size = assembly_resource->symbol_data.size; - - assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; - assembly_resource->resource.free_data = assembly_resource_symbol_owned_data; + bundled_resource_add_free_func ((MonoBundledResource *)assembly_resource, free_func, free_data); } assembly_resource->assembly.name = name; assembly_resource->assembly.data = data; @@ -406,66 +427,60 @@ mono_bundled_resources_add_assembly_resource (const char *id, const char *name, } void -mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data) +mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data) { // Check if assembly dll counterpart had been added via mono_register_bundled_assemblies - MonoBundledAssemblyResource *assembly_resource = mono_bundled_resources_get_assembly_resource (id); + MonoBundledAssemblyResource *assembly_resource = bundled_resources_get_assembly_resource (id); if (!assembly_resource) { assembly_resource = g_new0 (MonoBundledAssemblyResource, 1); assembly_resource->resource.type = MONO_BUNDLED_ASSEMBLY; assembly_resource->resource.id = id; - assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; - assembly_resource->resource.free_data = free_data; + assembly_resource->resource.free_func = bundled_resources_free_func; + bundled_resource_add_free_func ((MonoBundledResource *)assembly_resource, free_func, free_data); mono_bundled_resources_add ((MonoBundledResource **)&assembly_resource, 1); } else { // Ensure the MonoBundledSymbolData has not been initialized g_assert (!assembly_resource->symbol_data.data && assembly_resource->symbol_data.size == 0); - - MonoBundledAssemblyResource *assembly_resource_assembly_owned_data = g_new0 (MonoBundledAssemblyResource, 1); - assembly_resource_assembly_owned_data->resource.id = assembly_resource->resource.id; - assembly_resource_assembly_owned_data->resource.free_bundled_resource_func = assembly_resource->resource.free_bundled_resource_func; - assembly_resource_assembly_owned_data->resource.free_data = assembly_resource->resource.free_data; - assembly_resource_assembly_owned_data->assembly.name = assembly_resource->assembly.name; - assembly_resource_assembly_owned_data->assembly.data = assembly_resource->assembly.data; - assembly_resource_assembly_owned_data->assembly.size = assembly_resource->assembly.size; - - assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; - assembly_resource->resource.free_data = assembly_resource_assembly_owned_data; + bundled_resource_add_free_func ((MonoBundledResource *)assembly_resource, free_func, free_data); } assembly_resource->symbol_data.data = (const uint8_t *)data; assembly_resource->symbol_data.size = (uint32_t)size; } void -mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data) +mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data) { - MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = mono_bundled_resources_get_satellite_assembly_resource (id); + MonoBundledSatelliteAssemblyResource *satellite_assembly_resource = bundled_resources_get_satellite_assembly_resource (id); g_assert (!satellite_assembly_resource); + satellite_assembly_resource = g_new0 (MonoBundledSatelliteAssemblyResource, 1); satellite_assembly_resource->resource.type = MONO_BUNDLED_SATELLITE_ASSEMBLY; satellite_assembly_resource->resource.id = id; - satellite_assembly_resource->resource.free_bundled_resource_func = free_bundled_resource_func; - satellite_assembly_resource->resource.free_data = free_data; + satellite_assembly_resource->resource.free_func = bundled_resources_free_func; satellite_assembly_resource->satellite_assembly.name = name; satellite_assembly_resource->satellite_assembly.culture = culture; satellite_assembly_resource->satellite_assembly.data = data; satellite_assembly_resource->satellite_assembly.size = size; + + bundled_resource_add_free_func ((MonoBundledResource *)satellite_assembly_resource, free_func, free_data); mono_bundled_resources_add ((MonoBundledResource **)&satellite_assembly_resource, 1); } void -mono_bundled_resources_add_data_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data) +mono_bundled_resources_add_data_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, free_bundled_resource_func free_func, void *free_data) { - MonoBundledDataResource *data_resource = mono_bundled_resources_get_data_resource (id); + MonoBundledDataResource *data_resource = bundled_resources_get_data_resource (id); g_assert (!data_resource); + data_resource = g_new0 (MonoBundledDataResource, 1); data_resource->resource.type = MONO_BUNDLED_DATA; data_resource->resource.id = id; - data_resource->resource.free_bundled_resource_func = free_bundled_resource_func; - data_resource->resource.free_data = free_data; + data_resource->resource.free_func = bundled_resources_free_func; data_resource->data.name = name; data_resource->data.data = data; data_resource->data.size = size; + + bundled_resource_add_free_func ((MonoBundledResource *)data_resource, free_func, free_data); mono_bundled_resources_add ((MonoBundledResource **)&data_resource, 1); } @@ -481,7 +496,7 @@ mono_bundled_resources_add_data_resource (const char *id, const char *name, cons bool mono_bundled_resources_contains_assemblies (void) { - return bundle_contains_assemblies; + return bundled_resources_contains_assemblies; } //--------------------------------------------------------------------------------------- @@ -496,5 +511,5 @@ mono_bundled_resources_contains_assemblies (void) bool mono_bundled_resources_contains_satellite_assemblies (void) { - return bundle_contains_satellite_assemblies; + return bundled_resources_contains_satellite_assemblies; } diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index d6a7f6abdee5ca..7a83606850e1a8 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -72,11 +72,9 @@ extern void mono_register_icu_bundle (void); #endif /* INVARIANT_GLOBALIZATION */ #endif /* WASM_SINGLE_FILE */ -extern void mono_wasm_free_bundled_assembly_resource_func (void *resource); -extern void mono_wasm_free_bundled_satellite_assembly_resource_func (void *resource); -extern void mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); -extern void mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); -extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); +extern void mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *, void *), void *free_data); +extern void mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *, void *), void *free_data); +extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *, void *), void *free_data); extern const char* dotnet_wasi_getentrypointassemblyname(); int32_t mono_wasi_load_icu_data(const void* pData); @@ -119,6 +117,12 @@ typedef SgenDescriptor MonoGCDescriptor; #include "driver-gen.c" #endif +static void +bundled_resources_free_func (void *resource, void *free_data) +{ + free (free_data); +} + int mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned int size) { @@ -128,12 +132,12 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in char *new_name = strdup (name); //FIXME handle debugging assemblies with .exe extension strcpy (&new_name [len - 3], "dll"); - mono_bundled_resources_add_assembly_symbol_resource (new_name, data, size, mono_wasm_free_bundled_assembly_resource_func, NULL); + mono_bundled_resources_add_assembly_symbol_resource (new_name, data, size, bundled_resources_free_func, new_name); return 1; } const char *assembly_name = strdup (name); assert (assembly_name); - mono_bundled_resources_add_assembly_resource (assembly_name, assembly_name, data, size, mono_wasm_free_bundled_assembly_resource_func, NULL); + mono_bundled_resources_add_assembly_resource (assembly_name, assembly_name, data, size, bundled_resources_free_func, assembly_name); return mono_has_pdb_checksum ((char*)data, size); } @@ -143,19 +147,40 @@ char* gai_strerror(int code) { return result; } +static void +bundled_resources_free_slots_func (void *resource, void *free_data) +{ + if (free_data) { + void **slots = (void **)free_data; + for (int i = 0; slots [i]; i++) + free (slots [i]); + } +} + void mono_wasm_add_satellite_assembly (const char *name, const char *culture, const unsigned char *data, unsigned int size) { int id_len = strlen (culture) + 1 + strlen (name); // +1 is for the "/" char *id = (char *)malloc (sizeof (char) * (id_len + 1)); // +1 is for the terminating null character assert (id); + int num_char = snprintf (id, (id_len + 1), "%s/%s", culture, name); assert (num_char > 0 && num_char == id_len); + const char *satellite_assembly_name = strdup (name); assert (satellite_assembly_name); + const char *satellite_assembly_culture = strdup (culture); assert (satellite_assembly_culture); - mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, mono_wasm_free_bundled_satellite_assembly_resource_func, NULL); + + void **slots = malloc (sizeof (void *) * 4); + assert (slots); + slots [0] = id; + slots [1] = satellite_assembly_name; + slots [2] = satellite_assembly_culture; + slots [3] = NULL; + + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, bundled_resources_free_slots_func, slots); } static void *sysglobal_native_handle; diff --git a/src/mono/wasm/runtime/driver.c b/src/mono/wasm/runtime/driver.c index a90d7de8d9a4a0..218dbe3312c975 100644 --- a/src/mono/wasm/runtime/driver.c +++ b/src/mono/wasm/runtime/driver.c @@ -57,8 +57,6 @@ extern void mono_register_timezones_bundle (void); extern void mono_wasm_set_entrypoint_breakpoint (const char* assembly_name, int method_token); static void mono_wasm_init_finalizer_thread (void); -extern void mono_wasm_free_bundled_assembly_resource_func (void *resource); -extern void mono_wasm_free_bundled_satellite_assembly_resource_func (void *resource); extern void mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); extern void mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); @@ -186,6 +184,12 @@ mono_wasm_deregister_root (char *addr) #include "driver-gen.c" #endif +static void +bundled_resources_free_func (void *resource, void *free_data) +{ + free (free_data); +} + EMSCRIPTEN_KEEPALIVE int mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned int size) { @@ -194,28 +198,49 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in char *new_name = strdup (name); //FIXME handle debugging assemblies with .exe extension strcpy (&new_name [len - 3], "dll"); - mono_bundled_resources_add_assembly_symbol_resource (new_name, data, size, mono_wasm_free_bundled_assembly_resource_func, NULL); + mono_bundled_resources_add_assembly_symbol_resource (new_name, data, size, bundled_resources_free_func, new_name); return 1; } const char *assembly_name = strdup (name); assert (assembly_name); - mono_bundled_resources_add_assembly_resource (assembly_name, assembly_name, data, size, mono_wasm_free_bundled_assembly_resource_func, NULL); + mono_bundled_resources_add_assembly_resource (assembly_name, assembly_name, data, size, bundled_resources_free_func, assembly_name); return mono_has_pdb_checksum ((char*)data, size); } +static void +bundled_resources_free_slots_func (void *resource, void *free_data) +{ + if (free_data) { + void **slots = (void **)free_data; + for (int i = 0; slots [i]; i++) + free (slots [i]); + } +} + EMSCRIPTEN_KEEPALIVE void mono_wasm_add_satellite_assembly (const char *name, const char *culture, const unsigned char *data, unsigned int size) { int id_len = strlen (culture) + 1 + strlen (name); // +1 is for the "/" char *id = (char *)malloc (sizeof (char) * (id_len + 1)); // +1 is for the terminating null character assert (id); + int num_char = snprintf (id, (id_len + 1), "%s/%s", culture, name); assert (num_char > 0 && num_char == id_len); + const char *satellite_assembly_name = strdup (name); assert (satellite_assembly_name); + const char *satellite_assembly_culture = strdup (culture); assert (satellite_assembly_culture); - mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, mono_wasm_free_bundled_satellite_assembly_resource_func, NULL); + + void **slots = malloc (sizeof (void *) * 4); + assert (slots); + slots [0] = id; + slots [1] = satellite_assembly_name; + slots [2] = satellite_assembly_culture; + slots [3] = NULL; + + mono_bundled_resources_add_satellite_assembly_resource (id, satellite_assembly_name, satellite_assembly_culture, data, size, bundled_resources_free_slots_func, slots); } EMSCRIPTEN_KEEPALIVE void From 854c5ec249f7128b3f611e9d7905f39113bce800 Mon Sep 17 00:00:00 2001 From: lateralusX Date: Fri, 2 Jun 2023 15:19:43 +0200 Subject: [PATCH 102/110] Fix crash when output parameters are NULL. --- src/mono/mono/metadata/bundled-resources.c | 34 ++++++++++------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index fcc832986a3fb9..91f5e6216772de 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -272,20 +272,20 @@ bundled_resources_get_data_resource (const char *id) bool mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out, const uint8_t **symbol_data_out, uint32_t *symbol_size_out) { - *data_out = NULL; - *size_out = 0; - *symbol_data_out = NULL; - *size_out = 0; MonoBundledAssemblyResource *bundled_assembly_resource = bundled_resources_get_assembly_resource (id); if (!bundled_assembly_resource || !bundled_assembly_resource->assembly.data || bundled_assembly_resource->assembly.size == 0) return false; - *data_out = bundled_assembly_resource->assembly.data; - *size_out = bundled_assembly_resource->assembly.size; - *symbol_data_out = bundled_assembly_resource->symbol_data.data; - *symbol_size_out = bundled_assembly_resource->symbol_data.size; + if (data_out) + *data_out = bundled_assembly_resource->assembly.data; + if (size_out) + *size_out = bundled_assembly_resource->assembly.size; + if (symbol_data_out) + *symbol_data_out = bundled_assembly_resource->symbol_data.data; + if (symbol_size_out) + *symbol_size_out = bundled_assembly_resource->symbol_data.size; return true; } @@ -307,17 +307,16 @@ mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8 bool mono_bundled_resources_get_satellite_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out) { - *data_out = NULL; - *size_out = 0; - MonoBundledSatelliteAssemblyResource *bundled_satellite_assembly_resource = bundled_resources_get_satellite_assembly_resource (id); if (!bundled_satellite_assembly_resource || !bundled_satellite_assembly_resource->satellite_assembly.data || bundled_satellite_assembly_resource->satellite_assembly.size == 0) return false; - *data_out = bundled_satellite_assembly_resource->satellite_assembly.data; - *size_out = bundled_satellite_assembly_resource->satellite_assembly.size; + if (data_out) + *data_out = bundled_satellite_assembly_resource->satellite_assembly.data; + if (size_out) + *size_out = bundled_satellite_assembly_resource->satellite_assembly.size; return true; } @@ -339,17 +338,16 @@ mono_bundled_resources_get_satellite_assembly_resource_values (const char *id, c bool mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out) { - *data_out = NULL; - *size_out = 0; - MonoBundledDataResource *bundled_data_resource = bundled_resources_get_data_resource (id); if (!bundled_data_resource || !bundled_data_resource->data.data || bundled_data_resource->data.size == 0) return false; - *data_out = bundled_data_resource->data.data; - *size_out = bundled_data_resource->data.size; + if (data_out) + *data_out = bundled_data_resource->data.data; + if (size_out) + *size_out = bundled_data_resource->data.size; return true; } From 4379c3777b74c5d155508b662922f1726aed020f Mon Sep 17 00:00:00 2001 From: lateralusX Date: Fri, 2 Jun 2023 15:20:24 +0200 Subject: [PATCH 103/110] Fix WASM/WASI build errors. --- src/mono/wasi/runtime/driver.c | 12 ++++++------ src/mono/wasm/runtime/driver.c | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/mono/wasi/runtime/driver.c b/src/mono/wasi/runtime/driver.c index 7a83606850e1a8..d6a51f9be51b06 100644 --- a/src/mono/wasi/runtime/driver.c +++ b/src/mono/wasi/runtime/driver.c @@ -72,9 +72,9 @@ extern void mono_register_icu_bundle (void); #endif /* INVARIANT_GLOBALIZATION */ #endif /* WASM_SINGLE_FILE */ -extern void mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *, void *), void *free_data); -extern void mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *, void *), void *free_data); -extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *, void *), void *free_data); +extern void mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_func)(void *, void *), void *free_data); +extern void mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_func)(void *, void *), void *free_data); +extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_func)(void *, void *), void *free_data); extern const char* dotnet_wasi_getentrypointassemblyname(); int32_t mono_wasi_load_icu_data(const void* pData); @@ -135,7 +135,7 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in mono_bundled_resources_add_assembly_symbol_resource (new_name, data, size, bundled_resources_free_func, new_name); return 1; } - const char *assembly_name = strdup (name); + char *assembly_name = strdup (name); assert (assembly_name); mono_bundled_resources_add_assembly_resource (assembly_name, assembly_name, data, size, bundled_resources_free_func, assembly_name); return mono_has_pdb_checksum ((char*)data, size); @@ -167,10 +167,10 @@ mono_wasm_add_satellite_assembly (const char *name, const char *culture, const u int num_char = snprintf (id, (id_len + 1), "%s/%s", culture, name); assert (num_char > 0 && num_char == id_len); - const char *satellite_assembly_name = strdup (name); + char *satellite_assembly_name = strdup (name); assert (satellite_assembly_name); - const char *satellite_assembly_culture = strdup (culture); + char *satellite_assembly_culture = strdup (culture); assert (satellite_assembly_culture); void **slots = malloc (sizeof (void *) * 4); diff --git a/src/mono/wasm/runtime/driver.c b/src/mono/wasm/runtime/driver.c index 218dbe3312c975..23d01c27e6856e 100644 --- a/src/mono/wasm/runtime/driver.c +++ b/src/mono/wasm/runtime/driver.c @@ -57,9 +57,9 @@ extern void mono_register_timezones_bundle (void); extern void mono_wasm_set_entrypoint_breakpoint (const char* assembly_name, int method_token); static void mono_wasm_init_finalizer_thread (void); -extern void mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); -extern void mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); -extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_bundled_resource_func)(void *), void *free_data); +extern void mono_bundled_resources_add_assembly_resource (const char *id, const char *name, const uint8_t *data, uint32_t size, void (*free_func)(void *, void*), void *free_data); +extern void mono_bundled_resources_add_assembly_symbol_resource (const char *id, const uint8_t *data, uint32_t size, void (*free_func)(void *, void *), void *free_data); +extern void mono_bundled_resources_add_satellite_assembly_resource (const char *id, const char *name, const char *culture, const uint8_t *data, uint32_t size, void (*free_func)(void *, void*), void *free_data); #ifndef DISABLE_LEGACY_JS_INTEROP @@ -201,7 +201,7 @@ mono_wasm_add_assembly (const char *name, const unsigned char *data, unsigned in mono_bundled_resources_add_assembly_symbol_resource (new_name, data, size, bundled_resources_free_func, new_name); return 1; } - const char *assembly_name = strdup (name); + char *assembly_name = strdup (name); assert (assembly_name); mono_bundled_resources_add_assembly_resource (assembly_name, assembly_name, data, size, bundled_resources_free_func, assembly_name); return mono_has_pdb_checksum ((char*)data, size); @@ -227,10 +227,10 @@ mono_wasm_add_satellite_assembly (const char *name, const char *culture, const u int num_char = snprintf (id, (id_len + 1), "%s/%s", culture, name); assert (num_char > 0 && num_char == id_len); - const char *satellite_assembly_name = strdup (name); + char *satellite_assembly_name = strdup (name); assert (satellite_assembly_name); - const char *satellite_assembly_culture = strdup (culture); + char *satellite_assembly_culture = strdup (culture); assert (satellite_assembly_culture); void **slots = malloc (sizeof (void *) * 4); From 59ba15014573a45f51b5d24990b74203e498d447 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Mon, 5 Jun 2023 17:01:27 -0400 Subject: [PATCH 104/110] Align bundled resources registration struct --- ...-bundled-resource-preallocation-and-registration.template | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template index aea178a626f271..40192fc002b2b7 100644 --- a/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template +++ b/src/tasks/MonoTargetsTasks/Templates/mono-bundled-resource-preallocation-and-registration.template @@ -11,10 +11,13 @@ typedef enum { MONO_BUNDLED_RESOURCE_COUNT } MonoBundledResourceType; +typedef void (*free_bundled_resource_func)(void *, void*); + typedef struct _MonoBundledResource { MonoBundledResourceType type; const char *id; - void (*free_bundled_resource_func)(void *); + free_bundled_resource_func free_func; + void *free_data; } MonoBundledResource; typedef struct _MonoBundledData { From 560491a9bcfb1c7380438520695462732f7c0b52 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 6 Jun 2023 13:44:49 -0400 Subject: [PATCH 105/110] [task] Check PE metadata to determine resource type --- .../EmitBundleTask/EmitBundleBase.cs | 106 +++++++++++++----- 1 file changed, 76 insertions(+), 30 deletions(-) diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index 1edae210945ca3..654bf675112382 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -7,6 +7,8 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -63,22 +65,24 @@ public override bool Execute() List bundledResources = new(FilesToBundle.Length); foreach (ITaskItem bundledResource in FilesToBundle) { + var resourcePath = bundledResource.ItemSpec; + var registeredName = bundledResource.GetMetadata("RegisteredName"); if (string.IsNullOrEmpty(registeredName)) { string culture = bundledResource.GetMetadata("Culture"); if (!string.IsNullOrEmpty(culture)) { - registeredName = culture + "/" + Path.GetFileName(bundledResource.ItemSpec); + registeredName = culture + "/" + Path.GetFileName(resourcePath); } else { - registeredName = Path.GetFileName(bundledResource.ItemSpec); + registeredName = Path.GetFileName(resourcePath); } bundledResource.SetMetadata("RegisteredName", registeredName); } - string resourceDataSymbol = $"bundled_resource_{ToSafeSymbolName(TruncateEncodedHash(Utils.ComputeHashEx(bundledResource.ItemSpec, Utils.HashAlgorithmType.SHA256, Utils.HashEncodingType.Base64Safe), MaxEncodedHashLength))}"; + string resourceDataSymbol = $"bundled_resource_{ToSafeSymbolName(TruncateEncodedHash(Utils.ComputeHashEx(resourcePath, Utils.HashAlgorithmType.SHA256, Utils.HashEncodingType.Base64Safe), MaxEncodedHashLength))}"; if (resourceDataSymbolDictionary.ContainsKey(registeredName)) { throw new LogAsErrorException($"Multiple resources have the same RegisteredName '{registeredName}'. Ensure {nameof(FilesToBundle)} 'RegisteredName' metadata are set and unique."); @@ -99,6 +103,37 @@ public override bool Execute() } resourcesForDataSymbolDictionary.Add(resourceDataSymbol, resourcesWithDataSymbol); + bundledResource.SetMetadata("ResourceType", "DataResource"); + try + { + using FileStream resourceContents = File.OpenRead(resourcePath); + using PEReader resourcePEReader = new(resourceContents); + if (resourcePEReader.HasMetadata) + { + string? managedAssemblyCulture = null; + + var resourceMetadataReader = PEReaderExtensions.GetMetadataReader(resourcePEReader); + if (resourceMetadataReader.IsAssembly) + { + bundledResource.SetMetadata("ResourceType", "AssemblyResource"); + managedAssemblyCulture = resourceMetadataReader.GetString(resourceMetadataReader.GetAssemblyDefinition().Culture); + } + + bool isSatelliteAssembly = !string.IsNullOrEmpty(managedAssemblyCulture) && !managedAssemblyCulture!.Equals("neutral", StringComparison.OrdinalIgnoreCase); + if (resourcePath.EndsWith(".resources.dll", StringComparison.InvariantCultureIgnoreCase) || isSatelliteAssembly) + { + bundledResource.SetMetadata("ResourceType", "SatelliteAssemblyResource"); + if (isSatelliteAssembly) + bundledResource.SetMetadata("Culture", managedAssemblyCulture); + } + } + } + catch (BadImageFormatException e) + { + if (resourcePath.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)) + Log.LogMessage(MessageImportance.High, $"Resource '{resourcePath}' was interpreted with ResourceType 'DataResource' but has a '.dll' extension. Error: {e}"); + } + bundledResources.Add(bundledResource); } @@ -154,6 +189,7 @@ public override bool Execute() string resourceSymbols = GatherUniqueExportedResourceDataSymbols(bundledResources); var files = bundledResources.Select(bundledResource => { + var resourceType = bundledResource.GetMetadata("ResourceType"); var registeredName = bundledResource.GetMetadata("RegisteredName"); var resourceName = ToSafeSymbolName(registeredName, false); // Different timezone resources may have the same contents, use registered name to differentiate preallocated resources @@ -164,7 +200,7 @@ public override bool Execute() if (File.Exists(bundledResource.GetMetadata("SymbolFile"))) resourceSymbolName = ToSafeSymbolName(bundledResource.GetMetadata("SymbolFile")); - return (registeredName, resourceName, resourceDataSymbol, culture, resourceSymbolName); + return (resourceType, registeredName, resourceName, resourceDataSymbol, culture, resourceSymbolName); }).ToList(); Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); @@ -240,7 +276,7 @@ private string GatherUniqueExportedResourceDataSymbols(List uniqueDes return resourceSymbols.ToString(); } - private static void GenerateBundledResourcePreallocationAndRegistration(string resourceSymbols, string bundleRegistrationFunctionName, ICollection<(string registeredName, string resourceName, string resourceDataSymbol, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) + private static void GenerateBundledResourcePreallocationAndRegistration(string resourceSymbols, string bundleRegistrationFunctionName, ICollection<(string resourceType, string registeredName, string resourceName, string resourceDataSymbol, string culture, string? resourceSymbolName)> files, StreamWriter outputUtf8Writer) { List preallocatedSource = new (); @@ -261,34 +297,44 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r // Generate Preloaded MonoBundled*Resource structs string preloadedStruct; - if (!string.IsNullOrEmpty(tuple.culture)) - { - preloadedStruct = satelliteAssemblyTemplate; - preloadedStruct.Replace("%Culture%", tuple.culture); - resourceId = tuple.registeredName; - preallocatedSatelliteAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); - satelliteAssembliesCount += 1; - } - else if (tuple.registeredName.EndsWith(".dll")) + switch (tuple.resourceType) { - preloadedStruct = assemblyTemplate; - // Add associated symfile information to MonoBundledAssemblyResource structs - string preloadedSymbolData = ""; - if (!string.IsNullOrEmpty(tuple.resourceSymbolName)) + case "SatelliteAssemblyResource": { - preloadedSymbolData = $",\n{Utils.GetEmbeddedResource("mono-bundled-symbol.template") - .Replace("%ResourceSymbolName%", tuple.resourceSymbolName) - .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName!].ToString())}"; + preloadedStruct = satelliteAssemblyTemplate; + preloadedStruct.Replace("%Culture%", tuple.culture); + resourceId = tuple.registeredName; + preallocatedSatelliteAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); + satelliteAssembliesCount += 1; + break; + } + case "AssemblyResource": + { + preloadedStruct = assemblyTemplate; + // Add associated symfile information to MonoBundledAssemblyResource structs + string preloadedSymbolData = ""; + if (!string.IsNullOrEmpty(tuple.resourceSymbolName)) + { + preloadedSymbolData = $",\n{Utils.GetEmbeddedResource("mono-bundled-symbol.template") + .Replace("%ResourceSymbolName%", tuple.resourceSymbolName) + .Replace("%SymbolLen%", symbolDataLen[tuple.resourceSymbolName!].ToString())}"; + } + preloadedStruct = preloadedStruct.Replace("%MonoBundledSymbolData%", preloadedSymbolData); + preallocatedAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); + assembliesCount += 1; + break; + } + case "DataResource": + { + preloadedStruct = symbolDataTemplate; + preallocatedData.Add($" (MonoBundledResource *)&{tuple.resourceName}"); + dataCount += 1; + break; + } + default: + { + throw new Exception($"Unsupported ResourceType '{tuple.resourceType}' for Resource '{tuple.resourceName}' with registered name '{tuple.registeredName}'. Ensure that the resource's ResourceType metadata is populated."); } - preloadedStruct = preloadedStruct.Replace("%MonoBundledSymbolData%", preloadedSymbolData); - preallocatedAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); - assembliesCount += 1; - } - else - { - preloadedStruct = symbolDataTemplate; - preallocatedData.Add($" (MonoBundledResource *)&{tuple.resourceName}"); - dataCount += 1; } var resourceDataSymbol = tuple.resourceDataSymbol; From 5e41b812bdbc5d8b36816b4bd2420aa4519679e9 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 6 Jun 2023 17:01:17 -0400 Subject: [PATCH 106/110] Cleanup Satellite Assembly bundling --- .../EmitBundleTask/EmitBundleBase.cs | 65 ++++++++++--------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index 654bf675112382..f82e21b00e9c85 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -67,6 +67,38 @@ public override bool Execute() { var resourcePath = bundledResource.ItemSpec; + bundledResource.SetMetadata("ResourceType", "DataResource"); + try + { + using FileStream resourceContents = File.OpenRead(resourcePath); + using PEReader resourcePEReader = new(resourceContents); + if (resourcePEReader.HasMetadata) + { + string? managedAssemblyCulture = null; + + var resourceMetadataReader = PEReaderExtensions.GetMetadataReader(resourcePEReader); + if (resourceMetadataReader.IsAssembly) + { + bundledResource.SetMetadata("ResourceType", "AssemblyResource"); + managedAssemblyCulture = resourceMetadataReader.GetString(resourceMetadataReader.GetAssemblyDefinition().Culture); + } + + bool isSatelliteAssembly = !string.IsNullOrEmpty(managedAssemblyCulture) && !managedAssemblyCulture!.Equals("neutral", StringComparison.OrdinalIgnoreCase); + if (resourcePath.EndsWith(".resources.dll", StringComparison.InvariantCultureIgnoreCase) || isSatelliteAssembly) + { + bundledResource.SetMetadata("ResourceType", "SatelliteAssemblyResource"); + if (isSatelliteAssembly) { + bundledResource.SetMetadata("Culture", managedAssemblyCulture); + } + } + } + } + catch (BadImageFormatException e) + { + if (resourcePath.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)) + Log.LogMessage(MessageImportance.High, $"Resource '{resourcePath}' was interpreted with ResourceType 'DataResource' but has a '.dll' extension. Error: {e}"); + } + var registeredName = bundledResource.GetMetadata("RegisteredName"); if (string.IsNullOrEmpty(registeredName)) { @@ -103,37 +135,6 @@ public override bool Execute() } resourcesForDataSymbolDictionary.Add(resourceDataSymbol, resourcesWithDataSymbol); - bundledResource.SetMetadata("ResourceType", "DataResource"); - try - { - using FileStream resourceContents = File.OpenRead(resourcePath); - using PEReader resourcePEReader = new(resourceContents); - if (resourcePEReader.HasMetadata) - { - string? managedAssemblyCulture = null; - - var resourceMetadataReader = PEReaderExtensions.GetMetadataReader(resourcePEReader); - if (resourceMetadataReader.IsAssembly) - { - bundledResource.SetMetadata("ResourceType", "AssemblyResource"); - managedAssemblyCulture = resourceMetadataReader.GetString(resourceMetadataReader.GetAssemblyDefinition().Culture); - } - - bool isSatelliteAssembly = !string.IsNullOrEmpty(managedAssemblyCulture) && !managedAssemblyCulture!.Equals("neutral", StringComparison.OrdinalIgnoreCase); - if (resourcePath.EndsWith(".resources.dll", StringComparison.InvariantCultureIgnoreCase) || isSatelliteAssembly) - { - bundledResource.SetMetadata("ResourceType", "SatelliteAssemblyResource"); - if (isSatelliteAssembly) - bundledResource.SetMetadata("Culture", managedAssemblyCulture); - } - } - } - catch (BadImageFormatException e) - { - if (resourcePath.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)) - Log.LogMessage(MessageImportance.High, $"Resource '{resourcePath}' was interpreted with ResourceType 'DataResource' but has a '.dll' extension. Error: {e}"); - } - bundledResources.Add(bundledResource); } @@ -302,7 +303,7 @@ private static void GenerateBundledResourcePreallocationAndRegistration(string r case "SatelliteAssemblyResource": { preloadedStruct = satelliteAssemblyTemplate; - preloadedStruct.Replace("%Culture%", tuple.culture); + preloadedStruct = preloadedStruct.Replace("%Culture%", tuple.culture); resourceId = tuple.registeredName; preallocatedSatelliteAssemblies.Add($" (MonoBundledResource *)&{tuple.resourceName}"); satelliteAssembliesCount += 1; From 18afedac4f6c7a27b890af8da74724c7e05105c8 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Thu, 8 Jun 2023 17:38:32 -0400 Subject: [PATCH 107/110] Address Feedback --- src/mono/mono/component/mini-wasm-debugger.c | 11 ++++- src/mono/mono/metadata/mono-debug.c | 44 ++------------------ src/mono/wasm/wasm.proj | 13 +++--- 3 files changed, 19 insertions(+), 49 deletions(-) diff --git a/src/mono/mono/component/mini-wasm-debugger.c b/src/mono/mono/component/mini-wasm-debugger.c index 80e570f142b088..3ae717c2255e24 100644 --- a/src/mono/mono/component/mini-wasm-debugger.c +++ b/src/mono/mono/component/mini-wasm-debugger.c @@ -199,7 +199,16 @@ assembly_loaded (MonoProfiler *prof, MonoAssembly *assembly) return; } - if (mono_bundled_resources_get_assembly_resource_values(assembly->aname.name, NULL, NULL, NULL, NULL)) + if (mono_bundled_resources_get_assembly_resource_values (assembly->aname.name, NULL, NULL, NULL, NULL)) + return; + + size_t len = strlen (assembly->aname.name); + char *assembly_name_with_extension = (char *)g_malloc0 (sizeof(char) * (len + 5)); + memcpy (assembly_name_with_extension, assembly->aname.name, len); + memcpy (assembly_name_with_extension + len, ".dll\0", 5); + gboolean mono_bundle_has_assembly = mono_bundled_resources_get_assembly_resource_values (assembly_name_with_extension, NULL, NULL, NULL, NULL); + g_free (assembly_name_with_extension); + if (mono_bundle_has_assembly) return; if (mono_has_pdb_checksum ((char *) assembly_image->raw_data, assembly_image->raw_data_len)) { //if it's a release assembly we don't need to send to DebuggerProxy diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index 12cdc017422daa..eb096a0706efb1 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1082,29 +1082,12 @@ mono_register_symfile_for_assembly (const char *assembly_name, const mono_byte * static MonoDebugHandle * open_symfile_from_bundle (MonoImage *image) { - MonoDebugHandle *handle = NULL; const uint8_t *data = NULL; uint32_t size = 0; - if (mono_bundled_resources_get_assembly_resource_values (image->module_name, NULL, NULL, &data, &size)) - handle = mono_debug_open_image (image, data, size); - -#ifdef ENABLE_WEBCIL - size_t len = strlen (image->module_name); - char *extension = strrchr (image->module_name, '.'); - /* if image's module_name ends with an acceptable extension, check if theres a bundled resource with a .dll extension instead */ - if (extension && (!strcmp (extension, ".webcil") || !strcmp (extension, MONO_WEBCIL_IN_WASM_EXTENSION))) { - size_t n = extension - image->module_name; - char *module_name_dll_suffix = (char *)g_malloc0 (sizeof(char) * (n + 5)); - memcpy (module_name_dll_suffix, image->module_name, len); - memcpy (module_name_dll_suffix + n, ".dll\0", 5); - if (mono_bundled_resources_get_assembly_resource_values (module_name_dll_suffix, NULL, NULL, &data, &size)) - handle = mono_debug_open_image (image, data, size); - - g_free (module_name_dll_suffix); - } -#endif + if (!mono_bundled_resources_get_assembly_resource_values (image->module_name, NULL, NULL, &data, &size)) + return NULL; - return handle; + return mono_debug_open_image (image, data, size); } const mono_byte * @@ -1112,26 +1095,7 @@ mono_get_symfile_bytes_from_bundle (const char *assembly_name, int *size) { const uint8_t *symbol_data = NULL; uint32_t symbol_size = 0; - if (mono_bundled_resources_get_assembly_resource_values (assembly_name, NULL, NULL, &symbol_data, &symbol_size)) { - *size = symbol_size; - return (mono_byte *)symbol_data; - } - -#ifdef ENABLE_WEBCIL - if (!symbol_data) { - size_t len = strlen (assembly_name); - char *extension = strrchr (assembly_name, '.'); - /* if image's module_name ends with an acceptable extension, check if theres a bundled resource with a .dll extension instead */ - if (extension && (!strcmp (extension, ".webcil") || !strcmp (extension, MONO_WEBCIL_IN_WASM_EXTENSION))) { - size_t n = extension - assembly_name; - char *module_name_dll_suffix = (char *)g_malloc0 (sizeof(char) * (n + 5)); - memcpy (module_name_dll_suffix, assembly_name, len); - memcpy (module_name_dll_suffix + n, ".dll\0", 5); - mono_bundled_resources_get_assembly_resource_values (module_name_dll_suffix, NULL, NULL, &symbol_data, &symbol_size); - g_free (module_name_dll_suffix); - } - } -#endif + mono_bundled_resources_get_assembly_resource_values (assembly_name, NULL, NULL, &symbol_data, &symbol_size); *size = symbol_size; return (mono_byte *)symbol_data; diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index 39ff52ed2bdeb2..961c28ec9e07b9 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -92,17 +92,14 @@ <_WasmTimezonesInternal Include="$(_WasmTimezonesPath)\**\*.*" WasmRole="Timezone"/> - - - - <_WasmBundleTimezonesWithHashes Update="@(_WasmBundleTimezonesWithHashes)"> - /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmBundleTimezonesWithHashes.Identity)).Replace('\','/')) - + <_WasmTimezonesInternal Update="@(_WasmTimezonesInternal)"> + /usr/share/zoneinfo/$([MSBuild]::MakeRelative($(_WasmTimezonesPath), %(_WasmTimezonesInternal.Identity)).Replace('\','/')) + @@ -147,7 +144,7 @@ <_WasmBundleTimezonesToDelete Include="$(_WasmIntermediateOutputPath)*.c" /> <_WasmBundleTimezonesToDelete Remove="$(WasmObjDir)\$(_WasmTimezonesBundleSourceFile)" /> <_WasmBundleTimezonesToDelete Remove="%(BundledWasmTimezones.DestinationFile)" /> - <_WasmBundleTimezonesToDelete Remove="%(WasmBundleTimezonesObjects)" /> + <_WasmBundleTimezonesToDelete Remove="$(WasmObjDir)\%(WasmBundleTimezonesObjects)" /> From c2c69abe9c336b41fb0d2f8e7955a575fd386c1c Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Fri, 9 Jun 2023 13:12:09 -0400 Subject: [PATCH 108/110] Split MonoBundledAssemblyResource getter --- src/mono/mono/component/mini-wasm-debugger.c | 23 +++++++------ src/mono/mono/metadata/assembly.c | 2 +- .../metadata/bundled-resources-internals.h | 5 ++- src/mono/mono/metadata/bundled-resources.c | 33 ++++++++++++++++--- src/mono/mono/metadata/mono-debug.c | 5 +-- 5 files changed, 48 insertions(+), 20 deletions(-) diff --git a/src/mono/mono/component/mini-wasm-debugger.c b/src/mono/mono/component/mini-wasm-debugger.c index 3ae717c2255e24..b73db552ebc623 100644 --- a/src/mono/mono/component/mini-wasm-debugger.c +++ b/src/mono/mono/component/mini-wasm-debugger.c @@ -199,16 +199,14 @@ assembly_loaded (MonoProfiler *prof, MonoAssembly *assembly) return; } - if (mono_bundled_resources_get_assembly_resource_values (assembly->aname.name, NULL, NULL, NULL, NULL)) - return; + gboolean already_loaded = mono_bundled_resources_get_assembly_resource_values (assembly->aname.name, NULL, NULL); + if (!already_loaded && !g_str_has_suffix (assembly->aname.name, ".dll")) { + char *assembly_name_with_extension = g_strdup_printf ("%s.dll", assembly->aname.name); + already_loaded = mono_bundled_resources_get_assembly_resource_values (assembly_name_with_extension, NULL, NULL); + g_free (assembly_name_with_extension); + } - size_t len = strlen (assembly->aname.name); - char *assembly_name_with_extension = (char *)g_malloc0 (sizeof(char) * (len + 5)); - memcpy (assembly_name_with_extension, assembly->aname.name, len); - memcpy (assembly_name_with_extension + len, ".dll\0", 5); - gboolean mono_bundle_has_assembly = mono_bundled_resources_get_assembly_resource_values (assembly_name_with_extension, NULL, NULL, NULL, NULL); - g_free (assembly_name_with_extension); - if (mono_bundle_has_assembly) + if (already_loaded) return; if (mono_has_pdb_checksum ((char *) assembly_image->raw_data, assembly_image->raw_data_len)) { //if it's a release assembly we don't need to send to DebuggerProxy @@ -445,11 +443,12 @@ mono_wasm_send_dbg_command (int id, MdbgProtCommandSet command_set, int command, } else { - unsigned int assembly_size = 0; - unsigned int symfile_size = 0; const unsigned char* assembly_bytes = NULL; + unsigned int assembly_size = 0; + mono_bundled_resources_get_assembly_resource_values (assembly_name, &assembly_bytes, &assembly_size); const unsigned char* pdb_bytes = NULL; - mono_bundled_resources_get_assembly_resource_values (assembly_name, &assembly_bytes, &assembly_size, &pdb_bytes, &symfile_size); + unsigned int symfile_size = 0; + mono_bundled_resources_get_assembly_resource_symbol_values (assembly_name, &pdb_bytes, &symfile_size); m_dbgprot_buffer_init (&buf, assembly_size + symfile_size); m_dbgprot_buffer_add_byte_array (&buf, (uint8_t *) assembly_bytes, assembly_size); m_dbgprot_buffer_add_byte_array (&buf, (uint8_t *) pdb_bytes, symfile_size); diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index c5e5f6f8f7093a..4ba8463c685180 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -1459,7 +1459,7 @@ open_from_bundle_internal (MonoAssemblyLoadContext *alc, const char *filename, M const uint8_t *data = NULL; uint32_t size = 0; - if (mono_bundled_resources_get_assembly_resource_values (name, &data, &size, NULL, NULL)) + if (mono_bundled_resources_get_assembly_resource_values (name, &data, &size)) image = mono_image_open_from_data_internal (alc, (char *)data, size, FALSE, status, FALSE, name, NULL); g_free (name); diff --git a/src/mono/mono/metadata/bundled-resources-internals.h b/src/mono/mono/metadata/bundled-resources-internals.h index 5ece7ea507f6b5..1a8c03f4fb4e70 100644 --- a/src/mono/mono/metadata/bundled-resources-internals.h +++ b/src/mono/mono/metadata/bundled-resources-internals.h @@ -71,7 +71,10 @@ void mono_bundled_resources_add (MonoBundledResource **resources_to_bundle, uint32_t len); bool -mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out, const uint8_t **symbol_data_out, uint32_t *symbol_size_out); +mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); + +bool +mono_bundled_resources_get_assembly_resource_symbol_values (const char *id, const uint8_t **symbol_data_out, uint32_t *symbol_size_out); bool mono_bundled_resources_get_satellite_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out); diff --git a/src/mono/mono/metadata/bundled-resources.c b/src/mono/mono/metadata/bundled-resources.c index 91f5e6216772de..9eae4f5db8fa84 100644 --- a/src/mono/mono/metadata/bundled-resources.c +++ b/src/mono/mono/metadata/bundled-resources.c @@ -262,15 +262,13 @@ bundled_resources_get_data_resource (const char *id) // * id - Unique name of the resource // ** data_out - address to point at assembly byte data // ** size_out - address to point at assembly byte data size -// ** symbol_data_out - address to point at assembly symbol byte data -// ** symbol_size_out - address to point at assembly symbol byte data size // // Returns: -// bool - whether or not a valid MonoBundledAssemblyResource was found with key 'id' +// bool - whether or not a valid MonoBundledAssemblyResource->assembly was found with key 'id' // bool -mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out, const uint8_t **symbol_data_out, uint32_t *symbol_size_out) +mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out) { MonoBundledAssemblyResource *bundled_assembly_resource = bundled_resources_get_assembly_resource (id); if (!bundled_assembly_resource || @@ -282,6 +280,33 @@ mono_bundled_resources_get_assembly_resource_values (const char *id, const uint8 *data_out = bundled_assembly_resource->assembly.data; if (size_out) *size_out = bundled_assembly_resource->assembly.size; + + return true; +} + +//--------------------------------------------------------------------------------------- +// +// mono_bundled_resources_get_assembly_resource_symbol_values retrieves assembly symbol data associated +// with a key equivalent to the requested resource id if found. +// +// Arguments: +// * id - Unique name of the resource +// ** symbol_data_out - address to point at assembly symbol byte data +// ** symbol_size_out - address to point at assembly symbol byte data size +// +// Returns: +// bool - whether or not a valid MonoBundledAssemblyResource->symbol_data was found with key 'id' +// + +bool +mono_bundled_resources_get_assembly_resource_symbol_values (const char *id, const uint8_t **symbol_data_out, uint32_t *symbol_size_out) +{ + MonoBundledAssemblyResource *bundled_assembly_resource = bundled_resources_get_assembly_resource (id); + if (!bundled_assembly_resource || + !bundled_assembly_resource->symbol_data.data || + bundled_assembly_resource->symbol_data.size == 0) + return false; + if (symbol_data_out) *symbol_data_out = bundled_assembly_resource->symbol_data.data; if (symbol_size_out) diff --git a/src/mono/mono/metadata/mono-debug.c b/src/mono/mono/metadata/mono-debug.c index eb096a0706efb1..2472c62e62d33b 100644 --- a/src/mono/mono/metadata/mono-debug.c +++ b/src/mono/mono/metadata/mono-debug.c @@ -1084,7 +1084,7 @@ open_symfile_from_bundle (MonoImage *image) { const uint8_t *data = NULL; uint32_t size = 0; - if (!mono_bundled_resources_get_assembly_resource_values (image->module_name, NULL, NULL, &data, &size)) + if (!mono_bundled_resources_get_assembly_resource_symbol_values (image->module_name, &data, &size)) return NULL; return mono_debug_open_image (image, data, size); @@ -1095,7 +1095,8 @@ mono_get_symfile_bytes_from_bundle (const char *assembly_name, int *size) { const uint8_t *symbol_data = NULL; uint32_t symbol_size = 0; - mono_bundled_resources_get_assembly_resource_values (assembly_name, NULL, NULL, &symbol_data, &symbol_size); + if (!mono_bundled_resources_get_assembly_resource_symbol_values (assembly_name, &symbol_data, &symbol_size)) + return NULL; *size = symbol_size; return (mono_byte *)symbol_data; From 45d2f8bebaa29dfbb7769781a5df84f0140ad9f8 Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 13 Jun 2023 14:47:27 -0400 Subject: [PATCH 109/110] Addressing more feedback --- src/mono/wasi/build/WasiApp.Native.targets | 6 +- .../EmitBundleTask/EmitBundleBase.cs | 80 +++++++++---------- .../EmitBundleTask/EmitBundleObjectFiles.cs | 6 +- .../EmitBundleTask/EmitBundleSourceFiles.cs | 4 +- 4 files changed, 46 insertions(+), 50 deletions(-) diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 7108e7841ab483..93b461c0146c2f 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -359,19 +359,19 @@ BundleRegistrationFunctionName="mono_register_assemblies_bundle" BundleFile="$(_WasmAssembliesBundleObjectFile)" OutputDirectory="$(_WasmIntermediateOutputPath)"> - + <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmAssembliesBundleObjectFile)" /> - <_WasiObjectFilesForBundle Include="%(BundledWasmAssemblies.DestinationFile)" /> + <_WasiObjectFilesForBundle Include="%(_WasmBundledAssemblies.DestinationFile)" /> - + resourceDataSymbolDictionary = new(); + private readonly Dictionary _resourceDataSymbolDictionary = new(); - private Dictionary resourcesForDataSymbolDictionary= new(); + private readonly Dictionary _resourcesForDataSymbolDictionary = new(); + + private const string RegisteredName = "RegisteredName"; /// Truncate encoded hashes used in file names and symbols to 24 characters. /// Represents a 128-bit hash output encoded in base64 format. @@ -31,7 +33,7 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab /// Could have RegisteredName, otherwise it would be the filename. /// RegisteredName should be prefixed with namespace in form of unix like path. For example: "/usr/share/zoneinfo/" [Required] - public ITaskItem[] FilesToBundle { get; set; } = default!; + public ITaskItem[] FilesToBundle { get; set; } = Array.Empty(); /// /// The function to call before mono runtime initialization @@ -48,7 +50,8 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab /// /// Path to store build artifacts /// - public string OutputDirectory {get; set; } = default!; + [Required] + public string? OutputDirectory { get; set; } /// /// Resources that were bundled @@ -87,9 +90,8 @@ public override bool Execute() if (resourcePath.EndsWith(".resources.dll", StringComparison.InvariantCultureIgnoreCase) || isSatelliteAssembly) { bundledResource.SetMetadata("ResourceType", "SatelliteAssemblyResource"); - if (isSatelliteAssembly) { + if (isSatelliteAssembly) bundledResource.SetMetadata("Culture", managedAssemblyCulture); - } } } } @@ -99,41 +101,37 @@ public override bool Execute() Log.LogMessage(MessageImportance.High, $"Resource '{resourcePath}' was interpreted with ResourceType 'DataResource' but has a '.dll' extension. Error: {e}"); } - var registeredName = bundledResource.GetMetadata("RegisteredName"); + var registeredName = bundledResource.GetMetadata(RegisteredName); if (string.IsNullOrEmpty(registeredName)) { string culture = bundledResource.GetMetadata("Culture"); - if (!string.IsNullOrEmpty(culture)) - { - registeredName = culture + "/" + Path.GetFileName(resourcePath); - } - else - { - registeredName = Path.GetFileName(resourcePath); - } - bundledResource.SetMetadata("RegisteredName", registeredName); + registeredName = !string.IsNullOrEmpty(culture) ? culture + "/" + Path.GetFileName(resourcePath) : Path.GetFileName(resourcePath); + bundledResource.SetMetadata(RegisteredName, registeredName); } string resourceDataSymbol = $"bundled_resource_{ToSafeSymbolName(TruncateEncodedHash(Utils.ComputeHashEx(resourcePath, Utils.HashAlgorithmType.SHA256, Utils.HashEncodingType.Base64Safe), MaxEncodedHashLength))}"; - if (resourceDataSymbolDictionary.ContainsKey(registeredName)) + if (_resourceDataSymbolDictionary.ContainsKey(registeredName)) { - throw new LogAsErrorException($"Multiple resources have the same RegisteredName '{registeredName}'. Ensure {nameof(FilesToBundle)} 'RegisteredName' metadata are set and unique."); + throw new LogAsErrorException($"Multiple resources have the same {RegisteredName} '{registeredName}'. Ensure {nameof(FilesToBundle)} 'RegisteredName' metadata are set and unique."); } - resourceDataSymbolDictionary.Add(registeredName, resourceDataSymbol); + _resourceDataSymbolDictionary.Add(registeredName, resourceDataSymbol); - bundledResource.SetMetadata("DestinationFile", Path.Combine(OutputDirectory, resourceDataSymbol + GetDestinationFileExtension())); + string destinationFile = Path.Combine(OutputDirectory, resourceDataSymbol + GetDestinationFileExtension()); + bundledResource.SetMetadata("DestinationFile", destinationFile); string[] resourcesWithDataSymbol; - if (resourcesForDataSymbolDictionary.TryGetValue(resourceDataSymbol, out string[]? resourcesAlreadyWithDataSymbol)) + if (_resourcesForDataSymbolDictionary.TryGetValue(resourceDataSymbol, out string[]? resourcesAlreadyWithDataSymbol)) { - resourcesForDataSymbolDictionary.Remove(resourceDataSymbol); + _resourcesForDataSymbolDictionary.Remove(resourceDataSymbol); + Log.LogMessage(MessageImportance.Low, $"Resource '{registeredName}' has the same output destination file '{destinationFile}' as '{string.Join("', '", resourcesAlreadyWithDataSymbol)}'"); resourcesWithDataSymbol = resourcesAlreadyWithDataSymbol.Append(registeredName).ToArray(); } else { - resourcesWithDataSymbol = new[] {registeredName}; + resourcesWithDataSymbol = new[] { registeredName }; + Log.LogMessage(MessageImportance.Low, $"Resource '{registeredName}' is associated with output destination file '{destinationFile}'"); } - resourcesForDataSymbolDictionary.Add(resourceDataSymbol, resourcesWithDataSymbol); + _resourcesForDataSymbolDictionary.Add(resourceDataSymbol, resourcesWithDataSymbol); bundledResources.Add(bundledResource); } @@ -157,14 +155,15 @@ public override bool Execute() var inputFile = contentSourceFile.ItemSpec; var destinationFile = contentSourceFile.GetMetadata("DestinationFile"); - var registeredName = contentSourceFile.GetMetadata("RegisteredName"); + var registeredName = contentSourceFile.GetMetadata(RegisteredName); var count = Interlocked.Increment(ref verboseCount); Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, destinationFile); - var symbolName = resourceDataSymbolDictionary[registeredName]; - if (!Emit(destinationFile, (codeStream) => { + var symbolName = _resourceDataSymbolDictionary[registeredName]; + if (!EmitBundleFile(destinationFile, (codeStream) => + { using var inputStream = File.OpenRead(inputFile); using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); BundleFileToCSource(symbolName, inputStream, outputUtf8Writer); @@ -176,8 +175,8 @@ public override bool Execute() foreach (ITaskItem bundledResource in bundledResources) { - string registeredName = bundledResource.GetMetadata("RegisteredName"); - string resourceDataSymbol = resourceDataSymbolDictionary[registeredName]; + string registeredName = bundledResource.GetMetadata(RegisteredName); + string resourceDataSymbol = _resourceDataSymbolDictionary[registeredName]; bundledResource.SetMetadata("DataSymbol", $"{resourceDataSymbol}_data"); bundledResource.SetMetadata("DataLenSymbol", $"{resourceDataSymbol}_data_len"); bundledResource.SetMetadata("DataLenSymbolValue", symbolDataLen[resourceDataSymbol].ToString()); @@ -191,10 +190,10 @@ public override bool Execute() var files = bundledResources.Select(bundledResource => { var resourceType = bundledResource.GetMetadata("ResourceType"); - var registeredName = bundledResource.GetMetadata("RegisteredName"); + var registeredName = bundledResource.GetMetadata(RegisteredName); var resourceName = ToSafeSymbolName(registeredName, false); // Different timezone resources may have the same contents, use registered name to differentiate preallocated resources - var resourceDataSymbol = resourceDataSymbolDictionary[registeredName]; + var resourceDataSymbol = _resourceDataSymbolDictionary[registeredName]; string culture = bundledResource.GetMetadata("Culture"); string? resourceSymbolName = null; @@ -207,7 +206,7 @@ public override bool Execute() Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); // Generate source file to preallocate resources and register bundled resources - Emit(Path.Combine(OutputDirectory, BundleFile), (outputStream) => + EmitBundleFile(Path.Combine(OutputDirectory, BundleFile), (outputStream) => { using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); GenerateBundledResourcePreallocationAndRegistration(resourceSymbols, BundleRegistrationFunctionName, files, outputUtf8Writer); @@ -252,7 +251,7 @@ private static byte[] InitLookupTable() return lookup; } - public abstract bool Emit(string destinationFile, Action inputProvider); + public abstract bool EmitBundleFile(string destinationFile, Action writeToOutputStream); public abstract string GetDestinationFileExtension(); @@ -264,8 +263,8 @@ private string GatherUniqueExportedResourceDataSymbols(List uniqueDes HashSet resourcesAdded = new (); // Different Timezone resources may have the same contents foreach (var uniqueDestinationFile in uniqueDestinationFiles) { - string registeredName = uniqueDestinationFile.GetMetadata("RegisteredName"); - string resourceDataSymbol = resourceDataSymbolDictionary[registeredName]; + string registeredName = uniqueDestinationFile.GetMetadata(RegisteredName); + string resourceDataSymbol = _resourceDataSymbolDictionary[registeredName]; if (!resourcesAdded.Contains(resourceDataSymbol)) { resourceSymbols.AppendLine($"extern uint8_t {resourceDataSymbol}_data[];"); @@ -385,7 +384,7 @@ private void BundleFileToCSource(string symbolName, FileStream inputStream, Stre outputUtf8Writer.WriteLine("#include "); - string[] resourcesForDataSymbol = resourcesForDataSymbolDictionary[symbolName]; + string[] resourcesForDataSymbol = _resourcesForDataSymbolDictionary[symbolName]; outputUtf8Writer.WriteLine($"// Resource Registered Names: {string.Join(", ", resourcesForDataSymbol)}"); outputUtf8Writer.Write($"uint8_t {symbolName}_data[] = {{"); outputUtf8Writer.Flush(); @@ -443,12 +442,9 @@ private static string ToSafeSymbolName(string destinationFileName, bool filename } private static string TruncateEncodedHash(string encodedHash, int maxEncodedHashLength) - { - if (string.IsNullOrEmpty(encodedHash)) - return string.Empty; - - return encodedHash.Substring(0, Math.Min(encodedHash.Length, maxEncodedHashLength)); - } + => string.IsNullOrEmpty(encodedHash) + ? string.Empty + : encodedHash.Substring(0, Math.Min(encodedHash.Length, maxEncodedHashLength)); // Equivalent to "isalnum" private static bool IsAlphanumeric(char c) => c diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs index 15f0deea840fd2..3e0d8e0e4407d6 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleObjectFiles.cs @@ -22,7 +22,7 @@ public override bool Execute() return base.Execute(); } - public override bool Emit(string destinationFile, Action inputProvider) + public override bool EmitBundleFile(string destinationFile, Action EmitBundleFile) { if (Path.GetDirectoryName(destinationFile) is string destDir && !string.IsNullOrEmpty(destDir)) Directory.CreateDirectory(destDir); @@ -31,8 +31,8 @@ public override bool Emit(string destinationFile, Action inputProvider) ClangExecutable!, args: $"-xc -o \"{destinationFile}\" -c -", envVars: null, workingDir: null, silent: true, logStdErrAsMessage: false, - debugMessageImportance: MessageImportance.Low, label: null, - inputProvider); + debugMessageImportance: MessageImportance.Low, label: Path.GetFileName(destinationFile), + EmitBundleFile); if (exitCode != 0) { Log.LogError($"Failed to compile with exit code {exitCode}{Environment.NewLine}Output: {output}"); diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs index c67ba1c49f3b2f..fb2793d4f7ef0b 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleSourceFiles.cs @@ -10,11 +10,11 @@ // For emcc it's not possible, so we need to write the code to disk first and then compile it in MSBuild. public class EmitBundleSourceFiles : EmitBundleBase { - public override bool Emit(string destinationFile, Action inputProvider) + public override bool EmitBundleFile(string destinationFile, Action EmitBundleFile) { using (var fileStream = File.Create(destinationFile)) { - inputProvider(fileStream); + EmitBundleFile(fileStream); } return true; From b377ff4596818614082bcbcddba6bc8f18bb590c Mon Sep 17 00:00:00 2001 From: mdh1418 Date: Tue, 13 Jun 2023 15:17:00 -0400 Subject: [PATCH 110/110] Validate OutputDirectory --- src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index f4a93fd703be12..4378dc40461404 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -65,6 +65,12 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab public override bool Execute() { + if (!Directory.Exists(OutputDirectory)) + { + Log.LogError($"OutputDirectory={OutputDirectory} doesn't exist."); + return false; + } + List bundledResources = new(FilesToBundle.Length); foreach (ITaskItem bundledResource in FilesToBundle) {